From 87e6e4fe5c7e65cb69e70306f22de6daf16b6e14 Mon Sep 17 00:00:00 2001
From: Sylvain Gugger <35901082+sgugger@users.noreply.github.com>
Date: Mon, 27 Dec 2021 16:31:21 -0500
Subject: [PATCH] Doc styler v2 (#14950)
* New doc styler
* Fix issue with args at the start
* Code sample fixes
* Style code examples in MDX
* Fix more patterns
* Typo
* Typo
* More patterns
* Do without black for now
* Get more info in error
* Docstring style
* Re-enable check
* Quality
* Fix add_end_docstring decorator
* Fix docstring
---
.circleci/config.yml | 2 +-
Makefile | 4 +-
src/transformers/commands/lfs.py | 8 +-
src/transformers/commands/serving.py | 4 +-
src/transformers/configuration_utils.py | 51 +-
src/transformers/convert_graph_to_onnx.py | 3 +-
.../convert_pytorch_checkpoint_to_tf2.py | 2 +-
...ert_slow_tokenizers_checkpoints_to_fast.py | 2 +-
src/transformers/data/data_collator.py | 84 ++-
src/transformers/data/processors/glue.py | 8 +-
src/transformers/data/processors/squad.py | 7 +-
src/transformers/data/processors/utils.py | 4 +-
src/transformers/data/processors/xnli.py | 2 +-
src/transformers/debug_utils.py | 23 +-
src/transformers/deepspeed.py | 22 +-
.../feature_extraction_sequence_utils.py | 47 +-
src/transformers/feature_extraction_utils.py | 51 +-
src/transformers/file_utils.py | 5 +-
src/transformers/generation_beam_search.py | 34 +-
.../generation_flax_logits_process.py | 22 +-
src/transformers/generation_flax_utils.py | 31 +-
src/transformers/generation_logits_process.py | 63 +-
.../generation_stopping_criteria.py | 15 +-
src/transformers/generation_tf_utils.py | 127 ++--
src/transformers/generation_utils.py | 337 +++++----
src/transformers/image_utils.py | 15 +-
src/transformers/integrations.py | 15 +-
src/transformers/keras_callbacks.py | 5 +-
src/transformers/modelcard.py | 4 +-
src/transformers/modeling_flax_outputs.py | 198 +++---
src/transformers/modeling_flax_utils.py | 79 +-
src/transformers/modeling_outputs.py | 272 ++++---
src/transformers/modeling_tf_outputs.py | 225 +++---
src/transformers/modeling_tf_utils.py | 115 ++-
src/transformers/modeling_utils.py | 186 +++--
.../models/albert/configuration_albert.py | 33 +-
.../models/albert/modeling_albert.py | 71 +-
.../models/albert/modeling_flax_albert.py | 41 +-
.../models/albert/modeling_tf_albert.py | 77 +-
.../models/albert/tokenization_albert.py | 15 +-
.../models/albert/tokenization_albert_fast.py | 10 +-
src/transformers/models/auto/auto_factory.py | 115 ++-
.../models/auto/configuration_auto.py | 24 +-
src/transformers/models/auto/dynamic.py | 15 +-
.../models/auto/feature_extraction_auto.py | 23 +-
src/transformers/models/auto/modeling_auto.py | 2 +-
.../models/auto/modeling_flax_auto.py | 2 +-
.../models/auto/modeling_tf_auto.py | 2 +-
.../models/auto/processing_auto.py | 21 +-
.../models/auto/tokenization_auto.py | 37 +-
.../models/bart/configuration_bart.py | 28 +-
src/transformers/models/bart/modeling_bart.py | 188 +++--
.../models/bart/modeling_flax_bart.py | 115 ++-
.../models/bart/modeling_tf_bart.py | 123 ++--
.../models/bart/tokenization_bart.py | 5 +-
.../models/bart/tokenization_bart_fast.py | 5 +-
.../models/barthez/tokenization_barthez.py | 16 +-
.../barthez/tokenization_barthez_fast.py | 12 +-
.../models/bartpho/tokenization_bartpho.py | 12 +-
.../models/beit/configuration_beit.py | 15 +-
.../models/beit/feature_extraction_beit.py | 20 +-
src/transformers/models/beit/modeling_beit.py | 30 +-
.../models/beit/modeling_flax_beit.py | 30 +-
.../models/bert/configuration_bert.py | 33 +-
src/transformers/models/bert/modeling_bert.py | 8 +-
.../models/bert/modeling_flax_bert.py | 54 +-
.../models/bert/modeling_tf_bert.py | 107 +--
.../models/bert/tokenization_bert.py | 13 +-
.../models/bert/tokenization_bert_fast.py | 7 +-
.../configuration_bert_generation.py | 24 +-
.../modeling_bert_generation.py | 66 +-
.../tokenization_bert_generation.py | 8 +-
.../models/bertweet/tokenization_bertweet.py | 10 +-
.../models/big_bird/configuration_big_bird.py | 30 +-
.../models/big_bird/modeling_big_bird.py | 97 +--
.../models/big_bird/modeling_flax_big_bird.py | 54 +-
.../models/big_bird/tokenization_big_bird.py | 11 +-
.../big_bird/tokenization_big_bird_fast.py | 10 +-
.../configuration_bigbird_pegasus.py | 41 +-
.../modeling_bigbird_pegasus.py | 158 ++--
.../blenderbot/configuration_blenderbot.py | 27 +-
.../models/blenderbot/modeling_blenderbot.py | 175 +++--
.../blenderbot/modeling_flax_blenderbot.py | 92 +--
.../blenderbot/modeling_tf_blenderbot.py | 124 ++--
.../blenderbot/tokenization_blenderbot.py | 8 +-
.../tokenization_blenderbot_fast.py | 8 +-
.../configuration_blenderbot_small.py | 27 +-
.../modeling_blenderbot_small.py | 181 +++--
.../modeling_flax_blenderbot_small.py | 115 ++-
.../modeling_tf_blenderbot_small.py | 135 ++--
.../tokenization_blenderbot_small.py | 6 +-
.../models/byt5/tokenization_byt5.py | 11 +-
.../camembert/configuration_camembert.py | 6 +-
.../models/camembert/modeling_camembert.py | 49 +-
.../models/camembert/modeling_tf_camembert.py | 51 +-
.../camembert/tokenization_camembert.py | 16 +-
.../camembert/tokenization_camembert_fast.py | 11 +-
.../models/canine/configuration_canine.py | 17 +-
.../models/canine/modeling_canine.py | 81 ++-
.../models/canine/tokenization_canine.py | 6 +-
.../models/clip/configuration_clip.py | 50 +-
.../models/clip/feature_extraction_clip.py | 19 +-
src/transformers/models/clip/modeling_clip.py | 59 +-
.../models/clip/modeling_flax_clip.py | 67 +-
.../models/clip/modeling_tf_clip.py | 81 +--
.../models/clip/processing_clip.py | 39 +-
.../models/clip/tokenization_clip.py | 13 +-
.../models/clip/tokenization_clip_fast.py | 13 +-
.../models/convbert/configuration_convbert.py | 23 +-
.../models/convbert/modeling_convbert.py | 45 +-
.../models/convbert/modeling_tf_convbert.py | 70 +-
.../models/convbert/tokenization_convbert.py | 6 +-
.../convbert/tokenization_convbert_fast.py | 7 +-
.../models/cpm/tokenization_cpm.py | 11 +-
.../models/cpm/tokenization_cpm_fast.py | 11 +-
.../models/ctrl/configuration_ctrl.py | 17 +-
src/transformers/models/ctrl/modeling_ctrl.py | 76 +-
.../models/ctrl/modeling_tf_ctrl.py | 72 +-
.../models/ctrl/tokenization_ctrl.py | 4 +-
.../models/deberta/configuration_deberta.py | 35 +-
.../models/deberta/modeling_deberta.py | 60 +-
.../models/deberta/modeling_tf_deberta.py | 67 +-
.../models/deberta/tokenization_deberta.py | 3 +-
.../deberta/tokenization_deberta_fast.py | 7 +-
.../deberta_v2/configuration_deberta_v2.py | 25 +-
.../models/deberta_v2/modeling_deberta_v2.py | 60 +-
.../deberta_v2/modeling_tf_deberta_v2.py | 68 +-
.../deberta_v2/tokenization_deberta_v2.py | 11 +-
.../models/deit/configuration_deit.py | 16 +-
.../models/deit/feature_extraction_deit.py | 19 +-
src/transformers/models/deit/modeling_deit.py | 25 +-
.../models/detr/configuration_detr.py | 35 +-
.../models/detr/feature_extraction_detr.py | 72 +-
src/transformers/models/detr/modeling_detr.py | 168 +++--
.../distilbert/configuration_distilbert.py | 24 +-
.../models/distilbert/modeling_distilbert.py | 49 +-
.../distilbert/modeling_flax_distilbert.py | 20 +-
.../distilbert/modeling_tf_distilbert.py | 62 +-
.../distilbert/tokenization_distilbert.py | 7 +-
.../tokenization_distilbert_fast.py | 7 +-
.../models/dpr/configuration_dpr.py | 23 +-
src/transformers/models/dpr/modeling_dpr.py | 63 +-
.../models/dpr/modeling_tf_dpr.py | 73 +-
.../models/dpr/tokenization_dpr.py | 84 ++-
.../models/dpr/tokenization_dpr_fast.py | 84 ++-
.../models/electra/configuration_electra.py | 33 +-
.../models/electra/modeling_electra.py | 78 +-
.../models/electra/modeling_flax_electra.py | 53 +-
.../models/electra/modeling_tf_electra.py | 82 +--
.../models/electra/tokenization_electra.py | 7 +-
.../electra/tokenization_electra_fast.py | 7 +-
.../configuration_encoder_decoder.py | 22 +-
.../modeling_encoder_decoder.py | 108 ++-
.../modeling_flax_encoder_decoder.py | 119 ++--
.../modeling_tf_encoder_decoder.py | 91 ++-
.../models/flaubert/configuration_flaubert.py | 18 +-
.../models/flaubert/modeling_flaubert.py | 68 +-
.../models/flaubert/modeling_tf_flaubert.py | 55 +-
.../models/flaubert/tokenization_flaubert.py | 8 +-
.../models/fnet/configuration_fnet.py | 31 +-
src/transformers/models/fnet/modeling_fnet.py | 61 +-
.../models/fnet/tokenization_fnet.py | 14 +-
.../models/fnet/tokenization_fnet_fast.py | 10 +-
.../models/fsmt/configuration_fsmt.py | 22 +-
src/transformers/models/fsmt/modeling_fsmt.py | 79 +-
.../models/fsmt/tokenization_fsmt.py | 11 +-
.../models/funnel/configuration_funnel.py | 28 +-
.../models/funnel/modeling_funnel.py | 61 +-
.../models/funnel/modeling_tf_funnel.py | 74 +-
.../models/funnel/tokenization_funnel.py | 10 +-
.../models/funnel/tokenization_funnel_fast.py | 10 +-
.../models/gpt2/configuration_gpt2.py | 33 +-
.../models/gpt2/modeling_flax_gpt2.py | 30 +-
src/transformers/models/gpt2/modeling_gpt2.py | 100 +--
.../models/gpt2/modeling_tf_gpt2.py | 100 +--
.../models/gpt2/tokenization_gpt2.py | 10 +-
.../models/gpt2/tokenization_gpt2_fast.py | 10 +-
.../models/gpt_neo/configuration_gpt_neo.py | 27 +-
.../models/gpt_neo/modeling_flax_gpt_neo.py | 30 +-
.../models/gpt_neo/modeling_gpt_neo.py | 74 +-
.../models/gptj/configuration_gptj.py | 13 +-
.../models/gptj/modeling_flax_gptj.py | 30 +-
src/transformers/models/gptj/modeling_gptj.py | 57 +-
.../models/herbert/tokenization_herbert.py | 4 +-
.../herbert/tokenization_herbert_fast.py | 7 +-
.../models/hubert/configuration_hubert.py | 49 +-
.../models/hubert/modeling_hubert.py | 47 +-
.../models/hubert/modeling_tf_hubert.py | 59 +-
.../models/ibert/configuration_ibert.py | 31 +-
.../models/ibert/modeling_ibert.py | 57 +-
.../models/ibert/quant_modules.py | 13 +-
.../models/imagegpt/configuration_imagegpt.py | 17 +-
.../imagegpt/feature_extraction_imagegpt.py | 12 +-
.../models/imagegpt/modeling_imagegpt.py | 66 +-
.../models/layoutlm/configuration_layoutlm.py | 17 +-
.../models/layoutlm/modeling_layoutlm.py | 52 +-
.../models/layoutlm/modeling_tf_layoutlm.py | 57 +-
.../models/layoutlm/tokenization_layoutlm.py | 7 +-
.../layoutlm/tokenization_layoutlm_fast.py | 7 +-
.../layoutlmv2/configuration_layoutlmv2.py | 26 +-
.../feature_extraction_layoutlmv2.py | 20 +-
.../models/layoutlmv2/modeling_layoutlmv2.py | 46 +-
.../layoutlmv2/processing_layoutlmv2.py | 46 +-
.../layoutlmv2/tokenization_layoutlmv2.py | 102 ++-
.../tokenization_layoutlmv2_fast.py | 14 +-
.../models/layoutxlm/processing_layoutxlm.py | 49 +-
.../layoutxlm/tokenization_layoutxlm.py | 52 +-
.../layoutxlm/tokenization_layoutxlm_fast.py | 17 +-
.../models/led/configuration_led.py | 35 +-
src/transformers/models/led/modeling_led.py | 321 +++++----
.../models/led/modeling_tf_led.py | 170 ++---
.../models/led/tokenization_led.py | 7 +-
.../models/led/tokenization_led_fast.py | 7 +-
.../longformer/configuration_longformer.py | 21 +-
.../models/longformer/modeling_longformer.py | 281 ++++----
.../longformer/modeling_tf_longformer.py | 281 ++++----
.../longformer/tokenization_longformer.py | 4 +-
.../tokenization_longformer_fast.py | 4 +-
.../models/luke/configuration_luke.py | 17 +-
src/transformers/models/luke/modeling_luke.py | 169 ++---
.../models/luke/tokenization_luke.py | 82 +--
.../models/lxmert/configuration_lxmert.py | 18 +-
.../models/lxmert/modeling_lxmert.py | 138 ++--
.../models/lxmert/modeling_tf_lxmert.py | 91 +--
.../models/lxmert/tokenization_lxmert.py | 7 +-
.../models/lxmert/tokenization_lxmert_fast.py | 7 +-
.../models/m2m_100/configuration_m2m_100.py | 25 +-
.../models/m2m_100/modeling_m2m_100.py | 124 ++--
.../models/m2m_100/tokenization_m2m_100.py | 12 +-
.../models/marian/configuration_marian.py | 27 +-
.../models/marian/modeling_flax_marian.py | 86 +--
.../models/marian/modeling_marian.py | 130 ++--
.../models/marian/modeling_tf_marian.py | 94 +--
.../models/marian/tokenization_marian.py | 8 +-
.../models/mbart/configuration_mbart.py | 28 +-
.../models/mbart/modeling_flax_mbart.py | 118 ++-
.../models/mbart/modeling_mbart.py | 181 +++--
.../models/mbart/modeling_tf_mbart.py | 127 ++--
.../models/mbart/tokenization_mbart.py | 5 +-
.../models/mbart/tokenization_mbart_fast.py | 9 +-
.../models/mbart50/tokenization_mbart50.py | 8 +-
.../mbart50/tokenization_mbart50_fast.py | 7 +-
.../configuration_megatron_bert.py | 28 +-
.../megatron_bert/modeling_megatron_bert.py | 93 +--
.../models/mluke/tokenization_mluke.py | 85 +--
.../models/mmbt/configuration_mmbt.py | 6 +-
src/transformers/models/mmbt/modeling_mmbt.py | 43 +-
.../mobilebert/configuration_mobilebert.py | 22 +-
.../models/mobilebert/modeling_mobilebert.py | 53 +-
.../mobilebert/modeling_tf_mobilebert.py | 79 +-
.../mobilebert/tokenization_mobilebert.py | 7 +-
.../tokenization_mobilebert_fast.py | 7 +-
.../models/mpnet/configuration_mpnet.py | 21 +-
.../models/mpnet/modeling_mpnet.py | 52 +-
.../models/mpnet/modeling_tf_mpnet.py | 67 +-
.../models/mpnet/tokenization_mpnet.py | 14 +-
.../models/mpnet/tokenization_mpnet_fast.py | 12 +-
.../models/mt5/configuration_mt5.py | 20 +-
.../models/mt5/modeling_flax_mt5.py | 10 +-
src/transformers/models/mt5/modeling_mt5.py | 14 +-
.../models/mt5/modeling_tf_mt5.py | 14 +-
.../models/openai/configuration_openai.py | 41 +-
.../models/openai/modeling_openai.py | 71 +-
.../models/openai/modeling_tf_openai.py | 72 +-
.../models/openai/tokenization_openai.py | 4 +-
.../models/openai/tokenization_openai_fast.py | 4 +-
.../models/pegasus/configuration_pegasus.py | 28 +-
.../models/pegasus/modeling_flax_pegasus.py | 107 ++-
.../models/pegasus/modeling_pegasus.py | 176 ++---
.../models/pegasus/modeling_tf_pegasus.py | 119 ++--
.../models/pegasus/tokenization_pegasus.py | 12 +-
.../pegasus/tokenization_pegasus_fast.py | 11 +-
.../perceiver/configuration_perceiver.py | 16 +-
.../perceiver/feature_extraction_perceiver.py | 23 +-
.../models/perceiver/modeling_perceiver.py | 174 ++---
.../perceiver/tokenization_perceiver.py | 8 +-
.../models/phobert/tokenization_phobert.py | 10 +-
.../prophetnet/configuration_prophetnet.py | 14 +-
.../models/prophetnet/modeling_prophetnet.py | 182 ++---
.../prophetnet/tokenization_prophetnet.py | 15 +-
.../models/qdqbert/configuration_qdqbert.py | 17 +-
.../models/qdqbert/modeling_qdqbert.py | 88 +--
.../models/rag/configuration_rag.py | 25 +-
src/transformers/models/rag/modeling_rag.py | 245 +++----
.../models/rag/modeling_tf_rag.py | 195 ++---
src/transformers/models/rag/retrieval_rag.py | 35 +-
.../models/reformer/configuration_reformer.py | 58 +-
.../models/reformer/modeling_reformer.py | 114 +--
.../models/reformer/tokenization_reformer.py | 12 +-
.../reformer/tokenization_reformer_fast.py | 11 +-
.../models/rembert/configuration_rembert.py | 36 +-
.../models/rembert/modeling_rembert.py | 74 +-
.../models/rembert/modeling_tf_rembert.py | 95 +--
.../models/rembert/tokenization_rembert.py | 11 +-
.../rembert/tokenization_rembert_fast.py | 10 +-
.../retribert/configuration_retribert.py | 14 +-
.../models/retribert/modeling_retribert.py | 30 +-
.../retribert/tokenization_retribert.py | 7 +-
.../retribert/tokenization_retribert_fast.py | 7 +-
.../models/roberta/configuration_roberta.py | 15 +-
.../models/roberta/modeling_flax_roberta.py | 29 +-
.../models/roberta/modeling_roberta.py | 76 +-
.../models/roberta/modeling_tf_roberta.py | 93 +--
.../models/roberta/tokenization_roberta.py | 14 +-
.../roberta/tokenization_roberta_fast.py | 18 +-
.../models/roformer/configuration_roformer.py | 25 +-
.../models/roformer/modeling_roformer.py | 71 +-
.../models/roformer/modeling_tf_roformer.py | 72 +-
.../models/roformer/tokenization_roformer.py | 10 +-
.../roformer/tokenization_roformer_fast.py | 12 +-
.../segformer/configuration_segformer.py | 20 +-
.../segformer/feature_extraction_segformer.py | 12 +-
.../models/segformer/modeling_segformer.py | 19 +-
.../models/sew/configuration_sew.py | 41 +-
src/transformers/models/sew/modeling_sew.py | 37 +-
.../models/sew_d/configuration_sew_d.py | 41 +-
.../models/sew_d/modeling_sew_d.py | 44 +-
.../configuration_speech_encoder_decoder.py | 20 +-
.../modeling_speech_encoder_decoder.py | 127 ++--
.../configuration_speech_to_text.py | 24 +-
.../feature_extraction_speech_to_text.py | 20 +-
.../speech_to_text/modeling_speech_to_text.py | 115 +--
.../processing_speech_to_text.py | 40 +-
.../tokenization_speech_to_text.py | 8 +-
.../configuration_speech_to_text_2.py | 18 +-
.../modeling_speech_to_text_2.py | 75 +-
.../processing_speech_to_text_2.py | 36 +-
.../tokenization_speech_to_text_2.py | 4 +-
.../models/splinter/configuration_splinter.py | 17 +-
.../models/splinter/modeling_splinter.py | 52 +-
.../models/splinter/tokenization_splinter.py | 13 +-
.../splinter/tokenization_splinter_fast.py | 7 +-
.../squeezebert/configuration_squeezebert.py | 17 +-
.../squeezebert/modeling_squeezebert.py | 49 +-
.../squeezebert/tokenization_squeezebert.py | 7 +-
.../tokenization_squeezebert_fast.py | 7 +-
.../models/t5/configuration_t5.py | 24 +-
.../models/t5/modeling_flax_t5.py | 99 ++-
src/transformers/models/t5/modeling_t5.py | 99 ++-
src/transformers/models/t5/modeling_tf_t5.py | 105 +--
src/transformers/models/t5/tokenization_t5.py | 15 +-
.../models/t5/tokenization_t5_fast.py | 14 +-
.../models/tapas/configuration_tapas.py | 14 +-
.../models/tapas/modeling_tapas.py | 104 +--
.../models/tapas/modeling_tf_tapas.py | 107 +--
.../models/tapas/tokenization_tapas.py | 78 +-
.../transfo_xl/configuration_transfo_xl.py | 17 +-
.../transfo_xl/modeling_tf_transfo_xl.py | 76 +-
.../models/transfo_xl/modeling_transfo_xl.py | 74 +-
.../modeling_transfo_xl_utilities.py | 4 +-
.../transfo_xl/tokenization_transfo_xl.py | 15 +-
.../models/trocr/configuration_trocr.py | 21 +-
.../models/trocr/modeling_trocr.py | 75 +-
.../models/trocr/processing_trocr.py | 35 +-
.../unispeech/configuration_unispeech.py | 50 +-
.../models/unispeech/modeling_unispeech.py | 66 +-
.../configuration_unispeech_sat.py | 50 +-
.../unispeech_sat/modeling_unispeech_sat.py | 85 ++-
.../configuration_vision_encoder_decoder.py | 20 +-
.../modeling_flax_vision_encoder_decoder.py | 103 ++-
.../modeling_vision_encoder_decoder.py | 99 ++-
.../configuration_vision_text_dual_encoder.py | 18 +-
.../modeling_flax_vision_text_dual_encoder.py | 77 +-
.../modeling_vision_text_dual_encoder.py | 73 +-
.../processing_vision_text_dual_encoder.py | 43 +-
.../visual_bert/configuration_visual_bert.py | 35 +-
.../visual_bert/modeling_visual_bert.py | 65 +-
.../models/vit/configuration_vit.py | 17 +-
.../models/vit/feature_extraction_vit.py | 12 +-
.../models/vit/modeling_flax_vit.py | 21 +-
.../models/vit/modeling_tf_vit.py | 34 +-
src/transformers/models/vit/modeling_vit.py | 14 +-
.../models/wav2vec2/configuration_wav2vec2.py | 53 +-
.../wav2vec2/feature_extraction_wav2vec2.py | 38 +-
.../models/wav2vec2/modeling_flax_wav2vec2.py | 73 +-
.../models/wav2vec2/modeling_tf_wav2vec2.py | 59 +-
.../models/wav2vec2/modeling_wav2vec2.py | 83 ++-
.../models/wav2vec2/processing_wav2vec2.py | 42 +-
.../models/wav2vec2/tokenization_wav2vec2.py | 47 +-
.../tokenization_wav2vec2_phoneme.py | 12 +-
.../processing_wav2vec2_with_lm.py | 24 +-
.../models/wavlm/configuration_wavlm.py | 52 +-
.../models/wavlm/modeling_wavlm.py | 70 +-
.../models/xlm/configuration_xlm.py | 18 +-
.../models/xlm/modeling_tf_xlm.py | 68 +-
src/transformers/models/xlm/modeling_xlm.py | 71 +-
.../models/xlm/tokenization_xlm.py | 22 +-
.../configuration_xlm_prophetnet.py | 6 +-
.../xlm_prophetnet/modeling_xlm_prophetnet.py | 20 +-
.../tokenization_xlm_prophetnet.py | 12 +-
.../xlm_roberta/configuration_xlm_roberta.py | 6 +-
.../xlm_roberta/modeling_tf_xlm_roberta.py | 55 +-
.../xlm_roberta/modeling_xlm_roberta.py | 47 +-
.../xlm_roberta/tokenization_xlm_roberta.py | 12 +-
.../tokenization_xlm_roberta_fast.py | 11 +-
.../models/xlnet/configuration_xlnet.py | 38 +-
.../models/xlnet/modeling_tf_xlnet.py | 155 ++--
.../models/xlnet/modeling_xlnet.py | 194 ++---
.../models/xlnet/tokenization_xlnet.py | 15 +-
.../models/xlnet/tokenization_xlnet_fast.py | 14 +-
src/transformers/onnx/convert.py | 4 +-
src/transformers/optimization.py | 11 +-
src/transformers/optimization_tf.py | 14 +-
src/transformers/pipelines/__init__.py | 57 +-
.../pipelines/audio_classification.py | 21 +-
.../pipelines/automatic_speech_recognition.py | 25 +-
src/transformers/pipelines/base.py | 61 +-
src/transformers/pipelines/conversational.py | 34 +-
.../pipelines/feature_extraction.py | 15 +-
src/transformers/pipelines/fill_mask.py | 8 +-
.../pipelines/image_classification.py | 11 +-
.../pipelines/image_segmentation.py | 11 +-
.../pipelines/object_detection.py | 8 +-
.../pipelines/question_answering.py | 32 +-
.../pipelines/table_question_answering.py | 33 +-
.../pipelines/text2text_generation.py | 36 +-
.../pipelines/text_classification.py | 16 +-
src/transformers/pipelines/text_generation.py | 12 +-
.../pipelines/token_classification.py | 40 +-
.../pipelines/zero_shot_classification.py | 16 +-
src/transformers/testing_utils.py | 52 +-
src/transformers/tokenization_utils.py | 4 +-
src/transformers/tokenization_utils_base.py | 402 +++++------
src/transformers/tokenization_utils_fast.py | 16 +-
src/transformers/trainer.py | 144 ++--
src/transformers/trainer_callback.py | 35 +-
src/transformers/trainer_pt_utils.py | 69 +-
src/transformers/trainer_seq2seq.py | 10 +-
src/transformers/trainer_tf.py | 64 +-
src/transformers/trainer_utils.py | 3 +-
src/transformers/training_args.py | 211 +++---
src/transformers/training_args_seq2seq.py | 4 +-
src/transformers/training_args_tf.py | 53 +-
src/transformers/utils/fx.py | 9 +-
src/transformers/utils/logging.py | 2 +-
src/transformers/utils/notebook.py | 24 +-
utils/style_doc.py | 673 ++++++------------
437 files changed, 10810 insertions(+), 10811 deletions(-)
diff --git a/.circleci/config.yml b/.circleci/config.yml
index 109ff537fd..097a6dea80 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -848,7 +848,7 @@ jobs:
- run: isort --check-only examples tests src utils
- run: python utils/custom_init_isort.py --check_only
- run: flake8 examples tests src utils
-# - run: python utils/style_doc.py src/transformers docs/source --max_len 119 --check_only
+ - run: python utils/style_doc.py src/transformers docs/source --max_len 119 --check_only
check_repository_consistency:
working_directory: ~/transformers
diff --git a/Makefile b/Makefile
index 1a205d0e6d..75b7526625 100644
--- a/Makefile
+++ b/Makefile
@@ -48,13 +48,13 @@ quality:
isort --check-only $(check_dirs)
python utils/custom_init_isort.py --check_only
flake8 $(check_dirs)
-# python utils/style_doc.py src/transformers docs/source --max_len 119 --check_only
+ python utils/style_doc.py src/transformers docs/source --max_len 119 --check_only
# Format source code automatically and check is there are any problems left that need manual fixing
extra_style_checks:
python utils/custom_init_isort.py
-# python utils/style_doc.py src/transformers docs/source --max_len 119
+ python utils/style_doc.py src/transformers docs/source --max_len 119
# this target runs checks on all files and potentially modifies some of them
diff --git a/src/transformers/commands/lfs.py b/src/transformers/commands/lfs.py
index ac19549175..32c890af93 100644
--- a/src/transformers/commands/lfs.py
+++ b/src/transformers/commands/lfs.py
@@ -9,12 +9,8 @@ Spec is: github.com/git-lfs/git-lfs/blob/master/docs/custom-transfers.md
To launch debugger while developing:
``` [lfs "customtransfer.multipart"]
-
-path = /path/to/transformers/.env/bin/python
-
-args = -m debugpy --listen 5678 --wait-for-client /path/to/transformers/src/transformers/commands/transformers_cli.py
-lfs-multipart-upload ```
-"""
+ path = /path/to/transformers/.env/bin/python args = -m debugpy --listen 5678 --wait-for-client
+/path/to/transformers/src/transformers/commands/transformers_cli.py lfs-multipart-upload ``` """
import json
import os
diff --git a/src/transformers/commands/serving.py b/src/transformers/commands/serving.py
index fbe77cb8d9..9d53f948cf 100644
--- a/src/transformers/commands/serving.py
+++ b/src/transformers/commands/serving.py
@@ -214,9 +214,7 @@ class ServeCommand(BaseTransformersCLICommand):
async def forward(self, inputs=Body(None, embed=True)):
"""
- **inputs**:
- **attention_mask**:
- **tokens_type_ids**:
+ **inputs**: **attention_mask**: **tokens_type_ids**:
"""
# Check we don't have empty string
diff --git a/src/transformers/configuration_utils.py b/src/transformers/configuration_utils.py
index 25358fae42..407a46f78e 100755
--- a/src/transformers/configuration_utils.py
+++ b/src/transformers/configuration_utils.py
@@ -178,7 +178,8 @@ class PretrainedConfig(PushToHubMixin):
> Parameters for fine-tuning tasks
- architectures (`List[str]`, *optional*): Model architectures that can be used with the model pretrained weights.
+ architectures (`List[str]`, *optional*):
+ Model architectures that can be used with the model pretrained weights.
finetuning_task (`str`, *optional*):
Name of the task used to fine-tune the model. This can be used when converting from an original (TensorFlow
or PyTorch) checkpoint.
@@ -401,16 +402,14 @@ class PretrainedConfig(PushToHubMixin):
- Using `push_to_hub=True` will synchronize the repository you are pushing to with
- `save_directory`, which requires `save_directory` to be a local clone of the repo you are
- pushing to if it's an existing folder. Pass along `temp_dir=True` to use a temporary directory
- instead.
+ Using `push_to_hub=True` will synchronize the repository you are pushing to with `save_directory`,
+ which requires `save_directory` to be a local clone of the repo you are pushing to if it's an existing
+ folder. Pass along `temp_dir=True` to use a temporary directory instead.
kwargs:
- Additional key word arguments passed along to the
- [`~file_utils.PushToHubMixin.push_to_hub`] method.
+ Additional key word arguments passed along to the [`~file_utils.PushToHubMixin.push_to_hub`] method.
"""
if os.path.isfile(save_directory):
raise AssertionError(f"Provided path ({save_directory}) should be a directory, not a file")
@@ -433,8 +432,7 @@ class PretrainedConfig(PushToHubMixin):
@classmethod
def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig":
r"""
- Instantiate a [`PretrainedConfig`] (or a derived class) from a pretrained model
- configuration.
+ Instantiate a [`PretrainedConfig`] (or a derived class) from a pretrained model configuration.
Args:
pretrained_model_name_or_path (`str` or `os.PathLike`):
@@ -445,8 +443,7 @@ class PretrainedConfig(PushToHubMixin):
namespaced under a user or organization name, like `dbmdz/bert-base-german-cased`.
- a path to a *directory* containing a configuration file saved using the
[`~PretrainedConfig.save_pretrained`] method, e.g., `./my_model_directory/`.
- - a path or url to a saved configuration JSON *file*, e.g.,
- `./my_model_directory/configuration.json`.
+ - a path or url to a saved configuration JSON *file*, e.g., `./my_model_directory/configuration.json`.
cache_dir (`str` or `os.PathLike`, *optional*):
Path to a directory in which a downloaded pretrained model configuration should be cached if the
standard cache should not be used.
@@ -457,10 +454,11 @@ class PretrainedConfig(PushToHubMixin):
Whether or not to delete incompletely received file. Attempts to resume the download if such a file
exists.
proxies (`Dict[str, str]`, *optional*):
- A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request.
+ A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',
+ 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request.
use_auth_token (`str` or *bool*, *optional*):
- The token to use as HTTP bearer authorization for remote files. If `True`, will use the token
- generated when running `transformers-cli login` (stored in `~/.huggingface`).
+ The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated
+ when running `transformers-cli login` (stored in `~/.huggingface`).
revision(`str`, *optional*, defaults to `"main"`):
The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a
git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any
@@ -468,9 +466,9 @@ class PretrainedConfig(PushToHubMixin):
return_unused_kwargs (`bool`, *optional*, defaults to `False`):
If `False`, then this function returns just the final configuration object.
- If `True`, then this functions returns a `Tuple(config, unused_kwargs)` where *unused_kwargs*
- is a dictionary consisting of the key/value pairs whose keys are not configuration attributes: i.e.,
- the part of `kwargs` which has not been used to update `config` and is otherwise ignored.
+ If `True`, then this functions returns a `Tuple(config, unused_kwargs)` where *unused_kwargs* is a
+ dictionary consisting of the key/value pairs whose keys are not configuration attributes: i.e., the
+ part of `kwargs` which has not been used to update `config` and is otherwise ignored.
kwargs (`Dict[str, Any]`, *optional*):
The values in kwargs of any keys which are configuration attributes will be used to override the loaded
values. Behavior concerning key/value pairs whose keys are *not* configuration attributes is controlled
@@ -615,8 +613,7 @@ class PretrainedConfig(PushToHubMixin):
Args:
config_dict (`Dict[str, Any]`):
Dictionary that will be used to instantiate the configuration object. Such a dictionary can be
- retrieved from a pretrained checkpoint by leveraging the
- [`~PretrainedConfig.get_config_dict`] method.
+ retrieved from a pretrained checkpoint by leveraging the [`~PretrainedConfig.get_config_dict`] method.
kwargs (`Dict[str, Any]`):
Additional parameters from which to initialize the configuration object.
@@ -730,8 +727,8 @@ class PretrainedConfig(PushToHubMixin):
Args:
use_diff (`bool`, *optional*, defaults to `True`):
- If set to `True`, only the difference between the config instance and the default
- `PretrainedConfig()` is serialized to JSON string.
+ If set to `True`, only the difference between the config instance and the default `PretrainedConfig()`
+ is serialized to JSON string.
Returns:
`str`: String containing all the attributes that make up this configuration instance in JSON format.
@@ -750,8 +747,8 @@ class PretrainedConfig(PushToHubMixin):
json_file_path (`str` or `os.PathLike`):
Path to the JSON file in which this configuration instance's parameters will be saved.
use_diff (`bool`, *optional*, defaults to `True`):
- If set to `True`, only the difference between the config instance and the default
- `PretrainedConfig()` is serialized to JSON file.
+ If set to `True`, only the difference between the config instance and the default `PretrainedConfig()`
+ is serialized to JSON file.
"""
with open(json_file_path, "w", encoding="utf-8") as writer:
writer.write(self.to_json_string(use_diff=use_diff))
@@ -807,8 +804,8 @@ class PretrainedConfig(PushToHubMixin):
def dict_torch_dtype_to_str(self, d: Dict[str, Any]) -> None:
"""
Checks whether the passed dictionary has a *torch_dtype* key and if it's not None, converts torch.dtype to a
- string of just the type. For example, `torch.float32` get converted into *"float32"* string, which can
- then be stored in the json format.
+ string of just the type. For example, `torch.float32` get converted into *"float32"* string, which can then be
+ stored in the json format.
"""
if d.get("torch_dtype", None) is not None and not isinstance(d["torch_dtype"], str):
d["torch_dtype"] = str(d["torch_dtype"]).split(".")[1]
@@ -831,8 +828,8 @@ def get_configuration_file(
git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any
identifier allowed by git.
use_auth_token (`str` or *bool*, *optional*):
- The token to use as HTTP bearer authorization for remote files. If `True`, will use the token
- generated when running `transformers-cli login` (stored in `~/.huggingface`).
+ The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated
+ when running `transformers-cli login` (stored in `~/.huggingface`).
local_files_only (`bool`, *optional*, defaults to `False`):
Whether or not to only rely on local files and not to attempt to download any files.
diff --git a/src/transformers/convert_graph_to_onnx.py b/src/transformers/convert_graph_to_onnx.py
index 47fd6ca329..c1004b9d5a 100644
--- a/src/transformers/convert_graph_to_onnx.py
+++ b/src/transformers/convert_graph_to_onnx.py
@@ -348,7 +348,8 @@ def convert(
output: The path where the ONNX graph will be stored
opset: The actual version of the ONNX operator set to use
tokenizer: The name of the model to load for the pipeline, default to the model's name if not provided
- use_external_format: Split the model definition from its parameters to allow model bigger than 2GB (PyTorch only)
+ use_external_format:
+ Split the model definition from its parameters to allow model bigger than 2GB (PyTorch only)
pipeline_name: The kind of pipeline to instantiate (ner, question-answering, etc.)
model_kwargs: Keyword arguments to be forwarded to the model constructor
diff --git a/src/transformers/convert_pytorch_checkpoint_to_tf2.py b/src/transformers/convert_pytorch_checkpoint_to_tf2.py
index bcf69be478..4dc8daf283 100755
--- a/src/transformers/convert_pytorch_checkpoint_to_tf2.py
+++ b/src/transformers/convert_pytorch_checkpoint_to_tf2.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" Convert pytorch checkpoints to TensorFlow """
+""" Convert pytorch checkpoints to TensorFlow"""
import argparse
diff --git a/src/transformers/convert_slow_tokenizers_checkpoints_to_fast.py b/src/transformers/convert_slow_tokenizers_checkpoints_to_fast.py
index 208ecb640c..96458f4935 100755
--- a/src/transformers/convert_slow_tokenizers_checkpoints_to_fast.py
+++ b/src/transformers/convert_slow_tokenizers_checkpoints_to_fast.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" Convert slow tokenizers checkpoints in fast (serialization format of the `tokenizers` library) """
+""" Convert slow tokenizers checkpoints in fast (serialization format of the `tokenizers` library)"""
import argparse
import os
diff --git a/src/transformers/data/data_collator.py b/src/transformers/data/data_collator.py
index beeca57650..0b087b483d 100644
--- a/src/transformers/data/data_collator.py
+++ b/src/transformers/data/data_collator.py
@@ -219,12 +219,12 @@ class DataCollatorWithPadding:
Select a strategy to pad the returned sequences (according to the model's padding side and padding index)
among:
- - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
- sequence if provided).
- - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the
- maximum acceptable input length for the model if that argument is not provided.
- - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of
- different lengths).
+ - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single sequence
+ if provided).
+ - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
+ acceptable input length for the model if that argument is not provided.
+ - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different
+ lengths).
max_length (`int`, *optional*):
Maximum length of the returned list and optionally padding length (see above).
pad_to_multiple_of (`int`, *optional*):
@@ -271,12 +271,12 @@ class DataCollatorForTokenClassification(DataCollatorMixin):
Select a strategy to pad the returned sequences (according to the model's padding side and padding index)
among:
- - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
- sequence if provided).
- - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the
- maximum acceptable input length for the model if that argument is not provided.
- - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of
- different lengths).
+ - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single sequence
+ if provided).
+ - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
+ acceptable input length for the model if that argument is not provided.
+ - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different
+ lengths).
max_length (`int`, *optional*):
Maximum length of the returned list and optionally padding length (see above).
pad_to_multiple_of (`int`, *optional*):
@@ -526,12 +526,12 @@ class DataCollatorForSeq2Seq:
Select a strategy to pad the returned sequences (according to the model's padding side and padding index)
among:
- - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
- sequence is provided).
- - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the
- maximum acceptable input length for the model if that argument is not provided.
- - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of
- different lengths).
+ - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single sequence
+ is provided).
+ - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
+ acceptable input length for the model if that argument is not provided.
+ - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different
+ lengths).
max_length (`int`, *optional*):
Maximum length of the returned list and optionally padding length (see above).
pad_to_multiple_of (`int`, *optional*):
@@ -612,9 +612,9 @@ class DataCollatorForLanguageModeling(DataCollatorMixin):
tokenizer ([`PreTrainedTokenizer`] or [`PreTrainedTokenizerFast`]):
The tokenizer used for encoding the data.
mlm (`bool`, *optional*, defaults to `True`):
- Whether or not to use masked language modeling. If set to `False`, the labels are the same as the
- inputs with the padding tokens ignored (by setting them to -100). Otherwise, the labels are -100 for
- non-masked tokens and the value to predict for the masked token.
+ Whether or not to use masked language modeling. If set to `False`, the labels are the same as the inputs
+ with the padding tokens ignored (by setting them to -100). Otherwise, the labels are -100 for non-masked
+ tokens and the value to predict for the masked token.
mlm_probability (`float`, *optional*, defaults to 0.15):
The probability with which to (randomly) mask tokens in the input, when `mlm` is set to `True`.
pad_to_multiple_of (`int`, *optional*):
@@ -625,9 +625,8 @@ class DataCollatorForLanguageModeling(DataCollatorMixin):
For best performance, this data collator should be used with a dataset having items that are dictionaries or
- BatchEncoding, with the `"special_tokens_mask"` key, as returned by a
- [`PreTrainedTokenizer`] or a [`PreTrainedTokenizerFast`] with the
- argument `return_special_tokens_mask=True`.
+ BatchEncoding, with the `"special_tokens_mask"` key, as returned by a [`PreTrainedTokenizer`] or a
+ [`PreTrainedTokenizerFast`] with the argument `return_special_tokens_mask=True`.
"""
@@ -852,10 +851,9 @@ class DataCollatorForWholeWordMask(DataCollatorForLanguageModeling):
- This collator relies on details of the implementation of subword tokenization by
- [`BertTokenizer`], specifically that subword tokens are prefixed with *##*. For tokenizers
- that do not adhere to this scheme, this collator will produce an output that is roughly equivalent to
- [`.DataCollatorForLanguageModeling`].
+ This collator relies on details of the implementation of subword tokenization by [`BertTokenizer`], specifically
+ that subword tokens are prefixed with *##*. For tokenizers that do not adhere to this scheme, this collator will
+ produce an output that is roughly equivalent to [`.DataCollatorForLanguageModeling`].
"""
@@ -1234,13 +1232,13 @@ class DataCollatorForPermutationLanguageModeling(DataCollatorMixin):
The masked tokens to be predicted for a particular sequence are determined by the following algorithm:
0. Start from the beginning of the sequence by setting `cur_len = 0` (number of tokens processed so far).
- 1. Sample a `span_length` from the interval `[1, max_span_length]` (length of span of tokens to be
- masked)
+ 1. Sample a `span_length` from the interval `[1, max_span_length]` (length of span of tokens to be masked)
2. Reserve a context of length `context_length = span_length / plm_probability` to surround span to be
masked
- 3. Sample a starting point `start_index` from the interval `[cur_len, cur_len + context_length - span_length]` and mask tokens `start_index:start_index + span_length`
- 4. Set `cur_len = cur_len + context_length`. If `cur_len < max_len` (i.e. there are tokens remaining in
- the sequence to be processed), repeat from Step 1.
+ 3. Sample a starting point `start_index` from the interval `[cur_len, cur_len + context_length -
+ span_length]` and mask tokens `start_index:start_index + span_length`
+ 4. Set `cur_len = cur_len + context_length`. If `cur_len < max_len` (i.e. there are tokens remaining in the
+ sequence to be processed), repeat from Step 1.
"""
import torch
@@ -1331,13 +1329,13 @@ class DataCollatorForPermutationLanguageModeling(DataCollatorMixin):
The masked tokens to be predicted for a particular sequence are determined by the following algorithm:
0. Start from the beginning of the sequence by setting `cur_len = 0` (number of tokens processed so far).
- 1. Sample a `span_length` from the interval `[1, max_span_length]` (length of span of tokens to be
- masked)
+ 1. Sample a `span_length` from the interval `[1, max_span_length]` (length of span of tokens to be masked)
2. Reserve a context of length `context_length = span_length / plm_probability` to surround span to be
masked
- 3. Sample a starting point `start_index` from the interval `[cur_len, cur_len + context_length - span_length]` and mask tokens `start_index:start_index + span_length`
- 4. Set `cur_len = cur_len + context_length`. If `cur_len < max_len` (i.e. there are tokens remaining in
- the sequence to be processed), repeat from Step 1.
+ 3. Sample a starting point `start_index` from the interval `[cur_len, cur_len + context_length -
+ span_length]` and mask tokens `start_index:start_index + span_length`
+ 4. Set `cur_len = cur_len + context_length`. If `cur_len < max_len` (i.e. there are tokens remaining in the
+ sequence to be processed), repeat from Step 1.
"""
from random import randint
@@ -1439,13 +1437,13 @@ class DataCollatorForPermutationLanguageModeling(DataCollatorMixin):
The masked tokens to be predicted for a particular sequence are determined by the following algorithm:
0. Start from the beginning of the sequence by setting `cur_len = 0` (number of tokens processed so far).
- 1. Sample a `span_length` from the interval `[1, max_span_length]` (length of span of tokens to be
- masked)
+ 1. Sample a `span_length` from the interval `[1, max_span_length]` (length of span of tokens to be masked)
2. Reserve a context of length `context_length = span_length / plm_probability` to surround span to be
masked
- 3. Sample a starting point `start_index` from the interval `[cur_len, cur_len + context_length - span_length]` and mask tokens `start_index:start_index + span_length`
- 4. Set `cur_len = cur_len + context_length`. If `cur_len < max_len` (i.e. there are tokens remaining in
- the sequence to be processed), repeat from Step 1.
+ 3. Sample a starting point `start_index` from the interval `[cur_len, cur_len + context_length -
+ span_length]` and mask tokens `start_index:start_index + span_length`
+ 4. Set `cur_len = cur_len + context_length`. If `cur_len < max_len` (i.e. there are tokens remaining in the
+ sequence to be processed), repeat from Step 1.
"""
from random import randint
diff --git a/src/transformers/data/processors/glue.py b/src/transformers/data/processors/glue.py
index 24ff39ddb3..efd9529cf8 100644
--- a/src/transformers/data/processors/glue.py
+++ b/src/transformers/data/processors/glue.py
@@ -13,7 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" GLUE processors and helpers """
+""" GLUE processors and helpers"""
import os
import warnings
@@ -59,9 +59,9 @@ def glue_convert_examples_to_features(
output_mode: String indicating the output mode. Either `regression` or `classification`
Returns:
- If the `examples` input is a `tf.data.Dataset`, will return a `tf.data.Dataset` containing the
- task-specific features. If the input is a list of `InputExamples`, will return a list of task-specific
- `InputFeatures` which can be fed to the model.
+ If the `examples` input is a `tf.data.Dataset`, will return a `tf.data.Dataset` containing the task-specific
+ features. If the input is a list of `InputExamples`, will return a list of task-specific `InputFeatures` which
+ can be fed to the model.
"""
warnings.warn(DEPRECATION_WARNING.format("function"), FutureWarning)
diff --git a/src/transformers/data/processors/squad.py b/src/transformers/data/processors/squad.py
index 208ebe504f..5a2c288275 100644
--- a/src/transformers/data/processors/squad.py
+++ b/src/transformers/data/processors/squad.py
@@ -774,9 +774,10 @@ class SquadFeatures:
example_index: the index of the example
unique_id: The unique Feature identifier
paragraph_len: The length of the context
- token_is_max_context: List of booleans identifying which tokens have their maximum context in this feature object.
- If a token does not have their maximum context in this feature object, it means that another feature object
- has more information related to that token and should be prioritized over this feature for that token.
+ token_is_max_context:
+ List of booleans identifying which tokens have their maximum context in this feature object. If a token
+ does not have their maximum context in this feature object, it means that another feature object has more
+ information related to that token and should be prioritized over this feature for that token.
tokens: list of tokens corresponding to the input ids
token_to_orig_map: mapping between the tokens and the original text, needed in order to identify the answer.
start_position: start of the answer token index
diff --git a/src/transformers/data/processors/utils.py b/src/transformers/data/processors/utils.py
index bb008fe153..8bc39d0a73 100644
--- a/src/transformers/data/processors/utils.py
+++ b/src/transformers/data/processors/utils.py
@@ -248,8 +248,8 @@ class SingleSentenceClassificationProcessor(DataProcessor):
pad_on_left: If set to `True`, the examples will be padded on the left rather than on the right (default)
pad_token: Padding token
mask_padding_with_zero: If set to `True`, the attention mask will be filled by `1` for actual values
- and by `0` for padded values. If set to `False`, inverts it (`1` for padded values, `0` for
- actual values)
+ and by `0` for padded values. If set to `False`, inverts it (`1` for padded values, `0` for actual
+ values)
Returns:
If the `examples` input is a `tf.data.Dataset`, will return a `tf.data.Dataset` containing the
diff --git a/src/transformers/data/processors/xnli.py b/src/transformers/data/processors/xnli.py
index b5606bc030..4b27c309a1 100644
--- a/src/transformers/data/processors/xnli.py
+++ b/src/transformers/data/processors/xnli.py
@@ -13,7 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" XNLI utils (dataset loading and evaluation) """
+""" XNLI utils (dataset loading and evaluation)"""
import os
diff --git a/src/transformers/debug_utils.py b/src/transformers/debug_utils.py
index d876c40527..6685085549 100644
--- a/src/transformers/debug_utils.py
+++ b/src/transformers/debug_utils.py
@@ -43,14 +43,15 @@ class DebugUnderflowOverflow:
debug_overflow = DebugUnderflowOverflow(model)
```
- then run the training as normal and if `nan` or `inf` gets detected in at least one of the weight, input or
- output elements this module will throw an exception and will print `max_frames_to_save` frames that lead to this
- event, each frame reporting
+ then run the training as normal and if `nan` or `inf` gets detected in at least one of the weight, input or output
+ elements this module will throw an exception and will print `max_frames_to_save` frames that lead to this event,
+ each frame reporting
1. the fully qualified module name plus the class name whose `forward` was run
2. the absolute min and max value of all elements for each module weights, and the inputs and output
- For example, here is the header and the last few frames in detection report for `google/mt5-small` run in fp16 mixed precision :
+ For example, here is the header and the last few frames in detection report for `google/mt5-small` run in fp16
+ mixed precision :
```
Detected inf/nan during batch_number=0
@@ -77,8 +78,8 @@ class DebugUnderflowOverflow:
0.00e+00 inf output
```
- You can see here, that `T5DenseGatedGeluDense.forward` resulted in output activations, whose absolute max value
- was around 62.7K, which is very close to fp16's top limit of 64K. In the next frame we have `Dropout` which
+ You can see here, that `T5DenseGatedGeluDense.forward` resulted in output activations, whose absolute max value was
+ around 62.7K, which is very close to fp16's top limit of 64K. In the next frame we have `Dropout` which
renormalizes the weights, after it zeroed some of the elements, which pushes the absolute max value to more than
64K, and we get an overlow.
@@ -93,9 +94,9 @@ class DebugUnderflowOverflow:
debug_overflow = DebugUnderflowOverflow(model, max_frames_to_save=100)
```
- To validate that you have set up this debugging feature correctly, and you intend to use it in a training that may
- take hours to complete, first run it with normal tracing enabled for one of a few batches as explained in the next
- section.
+ To validate that you have set up this debugging feature correctly, and you intend to use it in a training that
+ may take hours to complete, first run it with normal tracing enabled for one of a few batches as explained in
+ the next section.
Mode 2. Specific batch absolute min/max tracing without detection
@@ -128,8 +129,8 @@ class DebugUnderflowOverflow:
**Performance**:
- As this module measures absolute `min`/``max` of each weight of the model on every forward it'll slow the
- training down. Therefore remember to turn it off once the debugging needs have been met.
+ As this module measures absolute `min`/``max` of each weight of the model on every forward it'll slow the training
+ down. Therefore remember to turn it off once the debugging needs have been met.
Args:
model (`nn.Module`):
diff --git a/src/transformers/deepspeed.py b/src/transformers/deepspeed.py
index 0680be888d..cb5621a5d4 100644
--- a/src/transformers/deepspeed.py
+++ b/src/transformers/deepspeed.py
@@ -42,12 +42,12 @@ class HfDeepSpeedConfig:
This object contains a DeepSpeed configuration dictionary and can be quickly queried for things like zero stage.
A `weakref` of this object is stored in the module's globals to be able to access the config from areas where
- things like the Trainer object is not available (e.g. `from_pretrained` and `_get_resized_embeddings`).
- Therefore it's important that this object remains alive while the program is still running.
+ things like the Trainer object is not available (e.g. `from_pretrained` and `_get_resized_embeddings`). Therefore
+ it's important that this object remains alive while the program is still running.
- [`Trainer`] uses the `HfTrainerDeepSpeedConfig` subclass instead. That subclass has logic to
- sync the configuration with values of [`TrainingArguments`] by replacing special placeholder
- values: `"auto"`. Without this special logic the DeepSpeed configuration is not modified in any way.
+ [`Trainer`] uses the `HfTrainerDeepSpeedConfig` subclass instead. That subclass has logic to sync the configuration
+ with values of [`TrainingArguments`] by replacing special placeholder values: `"auto"`. Without this special logic
+ the DeepSpeed configuration is not modified in any way.
Args:
config_file_or_dict (`Union[str, Dict]`): path to DeepSpeed config file or dict.
@@ -136,8 +136,8 @@ class HfDeepSpeedConfig:
def is_true(self, ds_key_long):
"""
- Returns `True`/``False` only if the value is set, always `False` otherwise. So use this method to ask the very specific question of whether the value is set to `True` (and it's not set to `False`` or
- isn't set).
+ Returns `True`/``False` only if the value is set, always `False` otherwise. So use this method to ask the very
+ specific question of whether the value is set to `True` (and it's not set to `False`` or isn't set).
"""
value = self.get_value(ds_key_long)
@@ -145,8 +145,8 @@ class HfDeepSpeedConfig:
def is_false(self, ds_key_long):
"""
- Returns `True`/``False` only if the value is set, always `False` otherwise. So use this method to ask the very specific question of whether the value is set to `False` (and it's not set to `True`` or
- isn't set).
+ Returns `True`/``False` only if the value is set, always `False` otherwise. So use this method to ask the very
+ specific question of whether the value is set to `False` (and it's not set to `True`` or isn't set).
"""
value = self.get_value(ds_key_long)
return False if value is None else not bool(value)
@@ -163,8 +163,8 @@ class HfDeepSpeedConfig:
class HfTrainerDeepSpeedConfig(HfDeepSpeedConfig):
"""
- The `HfTrainerDeepSpeedConfig` object is meant to be created during `TrainingArguments` object creation and has
- the same lifespan as the latter.
+ The `HfTrainerDeepSpeedConfig` object is meant to be created during `TrainingArguments` object creation and has the
+ same lifespan as the latter.
"""
def __init__(self, config_file_or_dict):
diff --git a/src/transformers/feature_extraction_sequence_utils.py b/src/transformers/feature_extraction_sequence_utils.py
index fbf2fc37ec..3b4a34ab1e 100644
--- a/src/transformers/feature_extraction_sequence_utils.py
+++ b/src/transformers/feature_extraction_sequence_utils.py
@@ -78,35 +78,36 @@ class SequenceFeatureExtractor(FeatureExtractionMixin):
Pad input values / input vectors or a batch of input values / input vectors up to predefined length or to the
max sequence length in the batch.
- Padding side (left/right) padding values are defined at the feature extractor level (with
- `self.padding_side`, `self.padding_value`)
+ Padding side (left/right) padding values are defined at the feature extractor level (with `self.padding_side`,
+ `self.padding_value`)
- If the `processed_features` passed are dictionary of numpy arrays, PyTorch tensors or TensorFlow tensors,
- the result will use the same type unless you provide a different tensor type with `return_tensors`. In
- the case of PyTorch tensors, you will lose the specific device of your tensors however.
+ If the `processed_features` passed are dictionary of numpy arrays, PyTorch tensors or TensorFlow tensors, the
+ result will use the same type unless you provide a different tensor type with `return_tensors`. In the case of
+ PyTorch tensors, you will lose the specific device of your tensors however.
Args:
processed_features ([`BatchFeature`], list of [`BatchFeature`], `Dict[str, List[float]]`, `Dict[str, List[List[float]]` or `List[Dict[str, List[float]]]`):
- Processed inputs. Can represent one input ([`BatchFeature`] or `Dict[str, List[float]]`) or a batch of input values / vectors (list of [`BatchFeature`],
- *Dict[str, List[List[float]]]* or *List[Dict[str, List[float]]]*) so you can use this method during
- preprocessing as well as in a PyTorch Dataloader collate function.
+ Processed inputs. Can represent one input ([`BatchFeature`] or `Dict[str, List[float]]`) or a batch of
+ input values / vectors (list of [`BatchFeature`], *Dict[str, List[List[float]]]* or *List[Dict[str,
+ List[float]]]*) so you can use this method during preprocessing as well as in a PyTorch Dataloader
+ collate function.
- Instead of `List[float]` you can have tensors (numpy arrays, PyTorch tensors or TensorFlow
- tensors), see the note above for the return type.
+ Instead of `List[float]` you can have tensors (numpy arrays, PyTorch tensors or TensorFlow tensors),
+ see the note above for the return type.
padding (`bool`, `str` or [`~file_utils.PaddingStrategy`], *optional*, defaults to `True`):
Select a strategy to pad the returned sequences (according to the model's padding side and padding
index) among:
- - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a
- single sequence if provided).
- - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the
- maximum acceptable input length for the model if that argument is not provided.
- - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of
- different lengths).
+ - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
+ sequence if provided).
+ - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
+ acceptable input length for the model if that argument is not provided.
+ - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different
+ lengths).
max_length (`int`, *optional*):
Maximum length of the returned list and optionally padding length (see above).
truncation (`bool`):
@@ -242,7 +243,9 @@ class SequenceFeatureExtractor(FeatureExtractionMixin):
Pad inputs (on left/right and up to predefined length or max length in the batch)
Args:
- processed_features: Dictionary of input values (`np.ndarray[float]`) / input vectors (`List[np.ndarray[float]]`) or batch of inputs values (`List[np.ndarray[int]]`) / input vectors (`List[np.ndarray[int]]`)
+ processed_features:
+ Dictionary of input values (`np.ndarray[float]`) / input vectors (`List[np.ndarray[float]]`) or batch
+ of inputs values (`List[np.ndarray[int]]`) / input vectors (`List[np.ndarray[int]]`)
max_length: maximum length of the returned list and optionally padding length (see below)
padding_strategy: PaddingStrategy to use for padding.
@@ -256,7 +259,8 @@ class SequenceFeatureExtractor(FeatureExtractionMixin):
pad_to_multiple_of: (optional) Integer if set will pad the sequence to a multiple of the provided value.
This is especially useful to enable the use of Tensor Core on NVIDIA hardware with compute capability
>= 7.5 (Volta), or on TPUs which benefit from having sequence lengths be a multiple of 128.
- return_attention_mask: (optional) Set to False to avoid returning attention mask (default: set to model specifics)
+ return_attention_mask:
+ (optional) Set to False to avoid returning attention mask (default: set to model specifics)
"""
required_input = processed_features[self.model_input_names[0]]
@@ -307,12 +311,15 @@ class SequenceFeatureExtractor(FeatureExtractionMixin):
Truncate inputs to predefined length or max length in the batch
Args:
- processed_features: Dictionary of input values (`np.ndarray[float]`) / input vectors (`List[np.ndarray[float]]`) or batch of inputs values (`List[np.ndarray[int]]`) / input vectors (`List[np.ndarray[int]]`)
+ processed_features:
+ Dictionary of input values (`np.ndarray[float]`) / input vectors (`List[np.ndarray[float]]`) or batch
+ of inputs values (`List[np.ndarray[int]]`) / input vectors (`List[np.ndarray[int]]`)
max_length: maximum length of the returned list and optionally padding length (see below)
pad_to_multiple_of: (optional) Integer if set will pad the sequence to a multiple of the provided value.
This is especially useful to enable the use of Tensor Core on NVIDIA hardware with compute capability
>= 7.5 (Volta), or on TPUs which benefit from having sequence lengths be a multiple of 128.
- truncation: (optional) Activates truncation to cut input sequences longer than `max_length` to `max_length`.
+ truncation:
+ (optional) Activates truncation to cut input sequences longer than `max_length` to `max_length`.
"""
if not truncation:
return processed_features
diff --git a/src/transformers/feature_extraction_utils.py b/src/transformers/feature_extraction_utils.py
index e96c8e23cd..f8f3cfcee3 100644
--- a/src/transformers/feature_extraction_utils.py
+++ b/src/transformers/feature_extraction_utils.py
@@ -54,8 +54,7 @@ PreTrainedFeatureExtractor = Union["SequenceFeatureExtractor"] # noqa: F821
class BatchFeature(UserDict):
r"""
- Holds the output of the [`~SequenceFeatureExtractor.pad`] and feature extractor specific
- `__call__` methods.
+ Holds the output of the [`~SequenceFeatureExtractor.pad`] and feature extractor specific `__call__` methods.
This class is derived from a python dictionary and can be used as a dictionary.
@@ -74,8 +73,8 @@ class BatchFeature(UserDict):
def __getitem__(self, item: str) -> Union[Any]:
"""
- If the key is a string, returns the value of the dict associated to `key` ('input_values',
- 'attention_mask', etc.).
+ If the key is a string, returns the value of the dict associated to `key` ('input_values', 'attention_mask',
+ etc.).
"""
if isinstance(item, str):
return self.data[item]
@@ -216,8 +215,8 @@ class FeatureExtractionMixin:
cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs
) -> PreTrainedFeatureExtractor:
r"""
- Instantiate a type of [`~feature_extraction_utils.FeatureExtractionMixin`] from a feature
- extractor, *e.g.* a derived class of [`SequenceFeatureExtractor`].
+ Instantiate a type of [`~feature_extraction_utils.FeatureExtractionMixin`] from a feature extractor, *e.g.* a
+ derived class of [`SequenceFeatureExtractor`].
Args:
pretrained_model_name_or_path (`str` or `os.PathLike`):
@@ -241,19 +240,20 @@ class FeatureExtractionMixin:
Whether or not to delete incompletely received file. Attempts to resume the download if such a file
exists.
proxies (`Dict[str, str]`, *optional*):
- A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request.
+ A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',
+ 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request.
use_auth_token (`str` or *bool*, *optional*):
- The token to use as HTTP bearer authorization for remote files. If `True`, will use the token
- generated when running `transformers-cli login` (stored in `~/.huggingface`).
+ The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated
+ when running `transformers-cli login` (stored in `~/.huggingface`).
revision(`str`, *optional*, defaults to `"main"`):
The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a
git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any
identifier allowed by git.
return_unused_kwargs (`bool`, *optional*, defaults to `False`):
- If `False`, then this function returns just the final feature extractor object. If `True`,
- then this functions returns a `Tuple(feature_extractor, unused_kwargs)` where *unused_kwargs* is a
- dictionary consisting of the key/value pairs whose keys are not feature extractor attributes: i.e., the
- part of `kwargs` which has not been used to update `feature_extractor` and is otherwise ignored.
+ If `False`, then this function returns just the final feature extractor object. If `True`, then this
+ functions returns a `Tuple(feature_extractor, unused_kwargs)` where *unused_kwargs* is a dictionary
+ consisting of the key/value pairs whose keys are not feature extractor attributes: i.e., the part of
+ `kwargs` which has not been used to update `feature_extractor` and is otherwise ignored.
kwargs (`Dict[str, Any]`, *optional*):
The values in kwargs of any keys which are feature extractor attributes will be used to override the
loaded values. Behavior concerning key/value pairs whose keys are *not* feature extractor attributes is
@@ -311,16 +311,14 @@ class FeatureExtractionMixin:
) -> Tuple[Dict[str, Any], Dict[str, Any]]:
"""
From a `pretrained_model_name_or_path`, resolve to a dictionary of parameters, to be used for instantiating a
- feature extractor of type [`~feature_extraction_utils.FeatureExtractionMixin`] using
- `from_dict`.
+ feature extractor of type [`~feature_extraction_utils.FeatureExtractionMixin`] using `from_dict`.
Parameters:
pretrained_model_name_or_path (`str` or `os.PathLike`):
The identifier of the pre-trained checkpoint from which we want the dictionary of parameters.
Returns:
- `Tuple[Dict, Dict]`: The dictionary(ies) that will be used to instantiate the feature extractor
- object.
+ `Tuple[Dict, Dict]`: The dictionary(ies) that will be used to instantiate the feature extractor object.
"""
cache_dir = kwargs.pop("cache_dir", None)
force_download = kwargs.pop("force_download", False)
@@ -398,8 +396,8 @@ class FeatureExtractionMixin:
@classmethod
def from_dict(cls, feature_extractor_dict: Dict[str, Any], **kwargs) -> PreTrainedFeatureExtractor:
"""
- Instantiates a type of [`~feature_extraction_utils.FeatureExtractionMixin`] from a Python
- dictionary of parameters.
+ Instantiates a type of [`~feature_extraction_utils.FeatureExtractionMixin`] from a Python dictionary of
+ parameters.
Args:
feature_extractor_dict (`Dict[str, Any]`):
@@ -410,8 +408,8 @@ class FeatureExtractionMixin:
Additional parameters from which to initialize the feature extractor object.
Returns:
- [`~feature_extraction_utils.FeatureExtractionMixin`]: The feature extractor object
- instantiated from those parameters.
+ [`~feature_extraction_utils.FeatureExtractionMixin`]: The feature extractor object instantiated from those
+ parameters.
"""
return_unused_kwargs = kwargs.pop("return_unused_kwargs", False)
@@ -447,16 +445,16 @@ class FeatureExtractionMixin:
@classmethod
def from_json_file(cls, json_file: Union[str, os.PathLike]) -> PreTrainedFeatureExtractor:
"""
- Instantiates a feature extractor of type [`~feature_extraction_utils.FeatureExtractionMixin`]
- from the path to a JSON file of parameters.
+ Instantiates a feature extractor of type [`~feature_extraction_utils.FeatureExtractionMixin`] from the path to
+ a JSON file of parameters.
Args:
json_file (`str` or `os.PathLike`):
Path to the JSON file containing the parameters.
Returns:
- A feature extractor of type [`~feature_extraction_utils.FeatureExtractionMixin`]: The
- feature_extractor object instantiated from that JSON file.
+ A feature extractor of type [`~feature_extraction_utils.FeatureExtractionMixin`]: The feature_extractor
+ object instantiated from that JSON file.
"""
with open(json_file, "r", encoding="utf-8") as reader:
text = reader.read()
@@ -468,8 +466,7 @@ class FeatureExtractionMixin:
Serializes this instance to a JSON string.
Returns:
- `str`: String containing all the attributes that make up this feature_extractor instance in JSON
- format.
+ `str`: String containing all the attributes that make up this feature_extractor instance in JSON format.
"""
dictionary = self.to_dict()
diff --git a/src/transformers/file_utils.py b/src/transformers/file_utils.py
index 1178489949..c7e1407321 100644
--- a/src/transformers/file_utils.py
+++ b/src/transformers/file_utils.py
@@ -855,7 +855,7 @@ def add_start_docstrings_to_model_forward(*docstr):
def add_end_docstrings(*docstr):
def docstring_decorator(fn):
- fn.__doc__ = fn.__doc__ + "".join(docstr)
+ fn.__doc__ = (fn.__doc__ if fn.__doc__ is not None else "") + "".join(docstr)
return fn
return docstring_decorator
@@ -1169,7 +1169,8 @@ PT_SPEECH_SEQ_CLASS_SAMPLE = r"""
>>> # audio file is decoded on the fly
>>> inputs = feature_extractor(dataset[0]["audio"]["array"], return_tensors="pt")
- >>> logits = model(**inputs).logits >>> predicted_class_ids = torch.argmax(logits, dim=-1)
+ >>> logits = model(**inputs).logits
+ >>> predicted_class_ids = torch.argmax(logits, dim=-1)
>>> predicted_label = model.config.id2label[predicted_class_ids]
>>> # compute loss - target_label is e.g. "down"
diff --git a/src/transformers/generation_beam_search.py b/src/transformers/generation_beam_search.py
index 663e8c31f0..3c4f259b00 100644
--- a/src/transformers/generation_beam_search.py
+++ b/src/transformers/generation_beam_search.py
@@ -29,8 +29,7 @@ PROCESS_INPUTS_DOCSTRING = r"""
Indices of input sequence tokens in the vocabulary.
Indices can be obtained using any class inheriting from [`PreTrainedTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
next_scores (`torch.FloatTensor` of shape `(batch_size, 2 * num_beams)`):
@@ -47,10 +46,10 @@ PROCESS_INPUTS_DOCSTRING = r"""
Return:
`UserDict`: A dictionary composed of the fields as defined above:
- - **next_beam_scores** (`torch.FloatTensor` of shape `(batch_size * num_beams)`) -- Updated
- scores of all non-finished beams.
- - **next_beam_tokens** (`torch.FloatTensor` of shape `(batch_size * num_beams)`) -- Next tokens
- to be added to the non-finished beam_hypotheses.
+ - **next_beam_scores** (`torch.FloatTensor` of shape `(batch_size * num_beams)`) -- Updated scores of all
+ non-finished beams.
+ - **next_beam_tokens** (`torch.FloatTensor` of shape `(batch_size * num_beams)`) -- Next tokens to be added
+ to the non-finished beam_hypotheses.
- **next_beam_indices** (`torch.FloatTensor` of shape `(batch_size * num_beams)`) -- Beam indices
indicating to which beam the next tokens shall be added.
@@ -62,8 +61,7 @@ FINALIZE_INPUTS_DOCSTRING = r"""
Indices of input sequence tokens in the vocabulary.
Indices can be obtained using any class inheriting from [`PreTrainedTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
final_beam_scores (`torch.FloatTensor` of shape `(batch_size * num_beams)`):
@@ -78,9 +76,9 @@ FINALIZE_INPUTS_DOCSTRING = r"""
The id of the *end-of-sequence* token.
Return:
- `torch.LongTensor` of shape `(batch_size * num_return_sequences, sequence_length)`: The generated
- sequences. The second dimension (sequence_length) is either equal to `max_length` or shorter if all
- batches finished early due to the `eos_token_id`.
+ `torch.LongTensor` of shape `(batch_size * num_return_sequences, sequence_length)`: The generated sequences.
+ The second dimension (sequence_length) is either equal to `max_length` or shorter if all batches finished early
+ due to the `eos_token_id`.
"""
@@ -121,9 +119,11 @@ class BeamSearchScorer(BeamScorer):
r"""
[`BeamScorer`] implementing standard beam search decoding.
- Adapted in part from [Facebook's XLM beam search code](https://github.com/facebookresearch/XLM/blob/9e6f6814d17be4fe5b15f2e6c43eb2b2d76daeb4/src/model/transformer.py#L529).
+ Adapted in part from [Facebook's XLM beam search
+ code](https://github.com/facebookresearch/XLM/blob/9e6f6814d17be4fe5b15f2e6c43eb2b2d76daeb4/src/model/transformer.py#L529).
- Reference for the diverse beam search algorithm and implementation [Ashwin Kalyan's DBS implementation](https://github.com/ashwinkalyan/dbs/blob/master/dbs/beam_utils.lua)
+ Reference for the diverse beam search algorithm and implementation [Ashwin Kalyan's DBS
+ implementation](https://github.com/ashwinkalyan/dbs/blob/master/dbs/beam_utils.lua)
Args:
batch_size (`int`):
@@ -133,8 +133,8 @@ class BeamSearchScorer(BeamScorer):
num_beams (`int`):
Number of beams for beam search.
device (`torch.device`):
- Defines the device type (*e.g.*, `"cpu"` or `"cuda"`) on which this instance of
- `BeamSearchScorer` will be allocated.
+ Defines the device type (*e.g.*, `"cpu"` or `"cuda"`) on which this instance of `BeamSearchScorer` will be
+ allocated.
length_penalty (`float`, *optional*, defaults to 1.0):
Exponential penalty to the length. 1.0 means no penalty. Set to values < 1.0 in order to encourage the
model to generate shorter sequences, to a value > 1.0 in order to encourage the model to produce longer
@@ -145,8 +145,8 @@ class BeamSearchScorer(BeamScorer):
The number of beam hypotheses that shall be returned upon calling
[`~transformer.BeamSearchScorer.finalize`].
num_beam_groups (`int`):
- Number of groups to divide `num_beams` into in order to ensure diversity among different groups of
- beams. See [this paper](https://arxiv.org/pdf/1610.02424.pdf) for more details.
+ Number of groups to divide `num_beams` into in order to ensure diversity among different groups of beams.
+ See [this paper](https://arxiv.org/pdf/1610.02424.pdf) for more details.
"""
def __init__(
diff --git a/src/transformers/generation_flax_logits_process.py b/src/transformers/generation_flax_logits_process.py
index 1b6bd4df6c..1d66953413 100644
--- a/src/transformers/generation_flax_logits_process.py
+++ b/src/transformers/generation_flax_logits_process.py
@@ -32,9 +32,8 @@ LOGITS_PROCESSOR_INPUTS_DOCSTRING = r"""
input_ids (`jnp.ndarray` of shape `(batch_size, sequence_length)`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`PreTrainedTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`PreTrainedTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
scores (`jnp.ndarray` of shape `(batch_size, config.vocab_size)`):
@@ -73,10 +72,9 @@ class FlaxLogitsWarper(ABC):
class FlaxLogitsProcessorList(list):
"""
- This class can be used to create a list of [`FlaxLogitsProcessor`] or
- [`FlaxLogitsWarper`] to subsequently process a `scores` input tensor. This class inherits
- from list and adds a specific *__call__* method to apply each [`FlaxLogitsProcessor`] or
- [`FlaxLogitsWarper`] to the inputs.
+ This class can be used to create a list of [`FlaxLogitsProcessor`] or [`FlaxLogitsWarper`] to subsequently process
+ a `scores` input tensor. This class inherits from list and adds a specific *__call__* method to apply each
+ [`FlaxLogitsProcessor`] or [`FlaxLogitsWarper`] to the inputs.
"""
@add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING)
@@ -117,13 +115,12 @@ class FlaxTemperatureLogitsWarper(FlaxLogitsWarper):
class FlaxTopPLogitsWarper(FlaxLogitsWarper):
"""
- [`LogitsWarper`] that performs top-p, i.e. restricting to top tokens summing to prob_cut_off <=
- prob_cut_off.
+ [`LogitsWarper`] that performs top-p, i.e. restricting to top tokens summing to prob_cut_off <= prob_cut_off.
Args:
top_p (`float`):
- If set to < 1, only the most probable tokens with probabilities that add up to `top_p` or higher are
- kept for generation.
+ If set to < 1, only the most probable tokens with probabilities that add up to `top_p` or higher are kept
+ for generation.
filter_value (`float`, *optional*, defaults to `-float("Inf")`):
All filtered values will be set to this float value.
min_tokens_to_keep (`int`, *optional*, defaults to 1):
@@ -219,8 +216,7 @@ class FlaxForcedBOSTokenLogitsProcessor(FlaxLogitsProcessor):
class FlaxForcedEOSTokenLogitsProcessor(FlaxLogitsProcessor):
r"""
- [`FlaxLogitsProcessor`] that enforces the specified token as the last generated token when
- `max_length` is reached.
+ [`FlaxLogitsProcessor`] that enforces the specified token as the last generated token when `max_length` is reached.
Args:
max_length (`int`):
diff --git a/src/transformers/generation_flax_utils.py b/src/transformers/generation_flax_utils.py
index 634540a215..8d2e10a7cd 100644
--- a/src/transformers/generation_flax_utils.py
+++ b/src/transformers/generation_flax_utils.py
@@ -118,8 +118,7 @@ class BeamSearchState:
class FlaxGenerationMixin:
"""
- A class containing all of the functions supporting generation, to be used as a mixin in
- [`FlaxPreTrainedModel`].
+ A class containing all of the functions supporting generation, to be used as a mixin in [`FlaxPreTrainedModel`].
"""
@staticmethod
@@ -148,8 +147,7 @@ class FlaxGenerationMixin:
def _adapt_logits_for_beam_search(self, logits):
"""
This function can be overwritten in the specific modeling_flax_.py classes to allow for custom beam
- search behavior. Note that the only model that overwrites this method is
- [`~transformes.FlaxMarianMTModel`].
+ search behavior. Note that the only model that overwrites this method is [`~transformes.FlaxMarianMTModel`].
"""
return logits
@@ -181,11 +179,12 @@ class FlaxGenerationMixin:
Generates sequences for models with a language modeling head. The method currently supports greedy decoding,
and, multinomial sampling.
- Apart from `input_ids`, all the arguments below will default to the value of the attribute of the same
- name inside the [`PretrainedConfig`] of the model. The default values indicated are the
- default values of those config.
+ Apart from `input_ids`, all the arguments below will default to the value of the attribute of the same name
+ inside the [`PretrainedConfig`] of the model. The default values indicated are the default values of those
+ config.
- Most of these parameters are explained in more detail in [this blog post](https://huggingface.co/blog/how-to-generate).
+ Most of these parameters are explained in more detail in [this blog
+ post](https://huggingface.co/blog/how-to-generate).
Parameters:
@@ -200,8 +199,8 @@ class FlaxGenerationMixin:
top_k (`int`, *optional*, defaults to 50):
The number of highest probability vocabulary tokens to keep for top-k-filtering.
top_p (`float`, *optional*, defaults to 1.0):
- If set to float < 1, only the most probable tokens with probabilities that add up to `top_p` or
- higher are kept for generation.
+ If set to float < 1, only the most probable tokens with probabilities that add up to `top_p` or higher
+ are kept for generation.
pad_token_id (`int`, *optional*):
The id of the *padding* token.
bos_token_id (`int`, *optional*):
@@ -213,8 +212,8 @@ class FlaxGenerationMixin:
decoder_start_token_id (`int`, *optional*):
If an encoder-decoder model starts decoding with a different token than *bos*, the id of that token.
trace (`bool`, *optional*, defaults to `True`):
- Whether to trace generation. Setting `trace=False` should only be used for debugging and will lead to
- a considerably slower runtime.
+ Whether to trace generation. Setting `trace=False` should only be used for debugging and will lead to a
+ considerably slower runtime.
params (`Dict[str, jnp.ndarray]`, *optional*):
Optionally the model parameters can be passed. Can be useful for parallelized generation.
model_kwargs:
@@ -327,8 +326,8 @@ class FlaxGenerationMixin:
self, top_k: int = None, top_p: float = None, temperature: float = None
) -> FlaxLogitsProcessorList:
"""
- This class returns a [`FlaxLogitsProcessorList`] list object that contains all relevant
- [`FlaxLogitsWarper`] instances used for multinomial sampling.
+ This class returns a [`FlaxLogitsProcessorList`] list object that contains all relevant [`FlaxLogitsWarper`]
+ instances used for multinomial sampling.
"""
# init warp parameters
@@ -359,8 +358,8 @@ class FlaxGenerationMixin:
forced_eos_token_id: int,
) -> FlaxLogitsProcessorList:
"""
- This class returns a [`FlaxLogitsProcessorList`] list object that contains all relevant
- [`FlaxLogitsProcessor`] instances used to modify the scores of the language model head.
+ This class returns a [`FlaxLogitsProcessorList`] list object that contains all relevant [`FlaxLogitsProcessor`]
+ instances used to modify the scores of the language model head.
"""
processors = FlaxLogitsProcessorList()
diff --git a/src/transformers/generation_logits_process.py b/src/transformers/generation_logits_process.py
index 8a9285f757..3e8a2f3cff 100644
--- a/src/transformers/generation_logits_process.py
+++ b/src/transformers/generation_logits_process.py
@@ -33,9 +33,8 @@ LOGITS_PROCESSOR_INPUTS_DOCSTRING = r"""
input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`BertTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`BertTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
scores (`torch.FloatTensor` of shape `(batch_size, config.vocab_size)`):
@@ -74,10 +73,9 @@ class LogitsWarper(ABC):
class LogitsProcessorList(list):
"""
- This class can be used to create a list of [`LogitsProcessor`] or
- [`LogitsWarper`] to subsequently process a `scores` input tensor. This class inherits from
- list and adds a specific *__call__* method to apply each [`LogitsProcessor`] or
- [`LogitsWarper`] to the inputs.
+ This class can be used to create a list of [`LogitsProcessor`] or [`LogitsWarper`] to subsequently process a
+ `scores` input tensor. This class inherits from list and adds a specific *__call__* method to apply each
+ [`LogitsProcessor`] or [`LogitsWarper`] to the inputs.
"""
@add_start_docstrings(LOGITS_PROCESSOR_INPUTS_DOCSTRING)
@@ -150,7 +148,8 @@ class RepetitionPenaltyLogitsProcessor(LogitsProcessor):
Args:
repetition_penalty (`float`):
- The parameter for repetition penalty. 1.0 means no penalty. See [this paper](https://arxiv.org/pdf/1909.05858.pdf) for more details.
+ The parameter for repetition penalty. 1.0 means no penalty. See [this
+ paper](https://arxiv.org/pdf/1909.05858.pdf) for more details.
"""
def __init__(self, penalty: float):
@@ -171,13 +170,12 @@ class RepetitionPenaltyLogitsProcessor(LogitsProcessor):
class TopPLogitsWarper(LogitsWarper):
"""
- [`LogitsWarper`] that performs top-p, i.e. restricting to top tokens summing to prob_cut_off <=
- prob_cut_off.
+ [`LogitsWarper`] that performs top-p, i.e. restricting to top tokens summing to prob_cut_off <= prob_cut_off.
Args:
top_p (`float`):
- If set to < 1, only the most probable tokens with probabilities that add up to `top_p` or higher are
- kept for generation.
+ If set to < 1, only the most probable tokens with probabilities that add up to `top_p` or higher are kept
+ for generation.
filter_value (`float`, *optional*, defaults to `-float("Inf")`):
All filtered values will be set to this float value.
min_tokens_to_keep (`int`, *optional*, defaults to 1):
@@ -278,7 +276,8 @@ def _calc_banned_ngram_tokens(
class NoRepeatNGramLogitsProcessor(LogitsProcessor):
r"""
- [`LogitsProcessor`] that enforces no repetition of n-grams. See [Fairseq](https://github.com/pytorch/fairseq/blob/a07cb6f40480928c9e0548b737aadd36ee66ac76/fairseq/sequence_generator.py#L345).
+ [`LogitsProcessor`] that enforces no repetition of n-grams. See
+ [Fairseq](https://github.com/pytorch/fairseq/blob/a07cb6f40480928c9e0548b737aadd36ee66ac76/fairseq/sequence_generator.py#L345).
Args:
ngram_size (`int`):
@@ -303,8 +302,8 @@ class NoRepeatNGramLogitsProcessor(LogitsProcessor):
class EncoderNoRepeatNGramLogitsProcessor(LogitsProcessor):
r"""
- [`LogitsProcessor`] that enforces no repetition of encoder input ids n-grams for the decoder ids.
- See [ParlAI](https://github.com/facebookresearch/ParlAI/blob/master/parlai/core/torch_generator_agent.py#L1350).
+ [`LogitsProcessor`] that enforces no repetition of encoder input ids n-grams for the decoder ids. See
+ [ParlAI](https://github.com/facebookresearch/ParlAI/blob/master/parlai/core/torch_generator_agent.py#L1350).
Args:
encoder_ngram_size (`int`):
@@ -471,16 +470,15 @@ class NoBadWordsLogitsProcessor(LogitsProcessor):
class PrefixConstrainedLogitsProcessor(LogitsProcessor):
r"""
- [`LogitsProcessor`] that enforces constrained generation and is useful for prefix-conditioned
- constrained generation. See [Autoregressive Entity Retrieval](https://arxiv.org/abs/2010.00904) for more
- information.
+ [`LogitsProcessor`] that enforces constrained generation and is useful for prefix-conditioned constrained
+ generation. See [Autoregressive Entity Retrieval](https://arxiv.org/abs/2010.00904) for more information.
Args:
prefix_allowed_tokens_fn: (`Callable[[int, torch.Tensor], List[int]]`):
This function constraints the beam search to allowed tokens only at each step. This function takes 2
- arguments `inputs_ids` and the batch ID `batch_id`. It has to return a list with the allowed
- tokens for the next generation step conditioned on the previously generated tokens `inputs_ids` and
- the batch ID `batch_id`.
+ arguments `inputs_ids` and the batch ID `batch_id`. It has to return a list with the allowed tokens for the
+ next generation step conditioned on the previously generated tokens `inputs_ids` and the batch ID
+ `batch_id`.
"""
def __init__(self, prefix_allowed_tokens_fn: Callable[[int, torch.Tensor], List[int]], num_beams: int):
@@ -498,20 +496,20 @@ class PrefixConstrainedLogitsProcessor(LogitsProcessor):
class HammingDiversityLogitsProcessor(LogitsProcessor):
r"""
- [`LogitsProcessor`] that enforces diverse beam search. Note that this logits processor is only
- effective for [`PreTrainedModel.group_beam_search`]. See [Diverse Beam Search: Decoding Diverse
- Solutions from Neural Sequence Models](https://arxiv.org/pdf/1610.02424.pdf) for more details.
+ [`LogitsProcessor`] that enforces diverse beam search. Note that this logits processor is only effective for
+ [`PreTrainedModel.group_beam_search`]. See [Diverse Beam Search: Decoding Diverse Solutions from Neural Sequence
+ Models](https://arxiv.org/pdf/1610.02424.pdf) for more details.
Args:
diversity_penalty (`float`):
This value is subtracted from a beam's score if it generates a token same as any beam from other group at a
particular time. Note that `diversity_penalty` is only effective if `group beam search` is enabled.
num_beams (`int`):
- Number of beams used for group beam search. See [this paper](https://arxiv.org/pdf/1610.02424.pdf) for
- more details.
+ Number of beams used for group beam search. See [this paper](https://arxiv.org/pdf/1610.02424.pdf) for more
+ details.
num_beam_groups (`int`):
- Number of groups to divide `num_beams` into in order to ensure diversity among different groups of
- beams. See [this paper](https://arxiv.org/pdf/1610.02424.pdf) for more details.
+ Number of groups to divide `num_beams` into in order to ensure diversity among different groups of beams.
+ See [this paper](https://arxiv.org/pdf/1610.02424.pdf) for more details.
"""
def __init__(self, diversity_penalty: float, num_beams: int, num_beam_groups: int):
@@ -579,8 +577,7 @@ class ForcedBOSTokenLogitsProcessor(LogitsProcessor):
class ForcedEOSTokenLogitsProcessor(LogitsProcessor):
r"""
- [`LogitsProcessor`] that enforces the specified token as the last generated token when
- `max_length` is reached.
+ [`LogitsProcessor`] that enforces the specified token as the last generated token when `max_length` is reached.
Args:
max_length (`int`):
@@ -604,9 +601,9 @@ class ForcedEOSTokenLogitsProcessor(LogitsProcessor):
class InfNanRemoveLogitsProcessor(LogitsProcessor):
r"""
- [`LogitsProcessor`] that removes all `nan` and `inf` values to avoid the generation
- method to fail. Note that using the logits processor should only be used if necessary since it can slow down the
- generation method. `max_length` is reached.
+ [`LogitsProcessor`] that removes all `nan` and `inf` values to avoid the generation method to fail. Note that using
+ the logits processor should only be used if necessary since it can slow down the generation method. `max_length` is
+ reached.
"""
def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
diff --git a/src/transformers/generation_stopping_criteria.py b/src/transformers/generation_stopping_criteria.py
index 27b92114bd..538cf330be 100644
--- a/src/transformers/generation_stopping_criteria.py
+++ b/src/transformers/generation_stopping_criteria.py
@@ -14,9 +14,8 @@ STOPPING_CRITERIA_INPUTS_DOCSTRING = r"""
input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`BertTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`BertTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
scores (`torch.FloatTensor` of shape `(batch_size, config.vocab_size)`):
@@ -41,8 +40,8 @@ class StoppingCriteria(ABC):
class MaxLengthCriteria(StoppingCriteria):
"""
- This class can be used to stop generation whenever the full generated number of tokens exceeds `max_length`.
- Keep in mind for decoder-only type of transformers, this will include the initial prompted tokens.
+ This class can be used to stop generation whenever the full generated number of tokens exceeds `max_length`. Keep
+ in mind for decoder-only type of transformers, this will include the initial prompted tokens.
Args:
max_length (`int`):
@@ -59,9 +58,9 @@ class MaxLengthCriteria(StoppingCriteria):
class MaxNewTokensCriteria(StoppingCriteria):
"""
- This class can be used to stop generation whenever the generated number of tokens exceeds `max_new_tokens`.
- Keep in mind for decoder-only type of transformers, this will **not** include the initial prompted tokens. This is
- very close to `MaxLengthCriteria` but ignores the number of initial tokens.
+ This class can be used to stop generation whenever the generated number of tokens exceeds `max_new_tokens`. Keep in
+ mind for decoder-only type of transformers, this will **not** include the initial prompted tokens. This is very
+ close to `MaxLengthCriteria` but ignores the number of initial tokens.
Args:
start_length (`int`):
diff --git a/src/transformers/generation_tf_utils.py b/src/transformers/generation_tf_utils.py
index 5cc103c7f7..5ac56dae1d 100644
--- a/src/transformers/generation_tf_utils.py
+++ b/src/transformers/generation_tf_utils.py
@@ -35,12 +35,12 @@ class TFGreedySearchDecoderOnlyOutput(ModelOutput):
Args:
sequences (`tf.Tensor` of shape `(batch_size, sequence_length)`):
- The generated sequences. The second dimension (sequence_length) is either equal to `max_length` or
- shorter if all batches finished early due to the `eos_token_id`.
+ The generated sequences. The second dimension (sequence_length) is either equal to `max_length` or shorter
+ if all batches finished early due to the `eos_token_id`.
scores (`tuple(tf.Tensor)` *optional*, returned when `output_scores=True` is passed or when `config.output_scores=True`):
Processed prediction scores of the language modeling head (scores for each vocabulary token before SoftMax)
- at each generation step. `(max_length-input_ids.shape[-1],)`-shaped tuple of `tf.Tensor` with
- each tensor of shape `(batch_size, config.vocab_size)`).
+ at each generation step. `(max_length-input_ids.shape[-1],)`-shaped tuple of `tf.Tensor` with each tensor
+ of shape `(batch_size, config.vocab_size)`).
attentions (`tuple(tuple(tf.Tensor))`, *optional*, returned when `output_attentions=True` is passed or `config.output_attentions=True`):
Tuple (one element for each generated token) of tuples (one element for each layer of the decoder) of
`tf.Tensor` of shape `(batch_size, num_heads, generated_length, sequence_length)`.
@@ -65,17 +65,18 @@ class TFGreedySearchEncoderDecoderOutput(ModelOutput):
Args:
sequences (`tf.Tensor` of shape `(batch_size, sequence_length)`):
- The generated sequences. The second dimension (sequence_length) is either equal to `max_length` or
- shorter if all batches finished early due to the `eos_token_id`.
+ The generated sequences. The second dimension (sequence_length) is either equal to `max_length` or shorter
+ if all batches finished early due to the `eos_token_id`.
scores (`tuple(tf.Tensor)` *optional*, returned when `output_scores=True` is passed or when `config.output_scores=True`):
Processed prediction scores of the language modeling head (scores for each vocabulary token before SoftMax)
at each generation step. `(max_length-1,)`-shaped tuple of `tf.Tensor` with each tensor of shape
`(batch_size, config.vocab_size)`).
encoder_attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer of the decoder) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer of the decoder) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
encoder_hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
decoder_attentions (`tuple(tuple(tf.Tensor))`, *optional*, returned when `output_attentions=True` is passed or `config.output_attentions=True`):
Tuple (one element for each generated token) of tuples (one element for each layer of the decoder) of
`tf.Tensor` of shape `(batch_size, num_heads, generated_length, sequence_length)`.
@@ -104,12 +105,12 @@ class TFSampleDecoderOnlyOutput(ModelOutput):
Args:
sequences (`tf.Tensor` of shape `(batch_size*num_return_sequences, sequence_length)`):
- The generated sequences. The second dimension (sequence_length) is either equal to `max_length` or
- shorter if all batches finished early due to the `eos_token_id`.
+ The generated sequences. The second dimension (sequence_length) is either equal to `max_length` or shorter
+ if all batches finished early due to the `eos_token_id`.
scores (`tuple(tf.Tensor)` *optional*, returned when `output_scores=True` is passed or when `config.output_scores=True`):
Processed prediction scores of the language modeling head (scores for each vocabulary token before SoftMax)
- at each generation step. `(max_length-input_ids.shape[-1],)`-shaped tuple of `tf.Tensor` with
- each tensor of shape `(batch_size*num_return_sequences, config.vocab_size)`).
+ at each generation step. `(max_length-input_ids.shape[-1],)`-shaped tuple of `tf.Tensor` with each tensor
+ of shape `(batch_size*num_return_sequences, config.vocab_size)`).
attentions (`tuple(tuple(tf.Tensor))`, *optional*, returned when `output_attentions=True` is passed or `config.output_attentions=True`):
Tuple (one element for each generated token) of tuples (one element for each layer of the decoder) of
`tf.Tensor` of shape `(num_return_sequences*batch_size, num_heads, generated_length, sequence_length)`.
@@ -134,18 +135,18 @@ class TFSampleEncoderDecoderOutput(ModelOutput):
Args:
sequences (`tf.Tensor` of shape `(batch_size*num_return_sequences, sequence_length)`):
- The generated sequences. The second dimension (sequence_length) is either equal to `max_length` or
- shorter if all batches finished early due to the `eos_token_id`.
+ The generated sequences. The second dimension (sequence_length) is either equal to `max_length` or shorter
+ if all batches finished early due to the `eos_token_id`.
scores (`tuple(tf.Tensor)` *optional*, returned when `output_scores=True` is passed or when `config.output_scores=True`):
Processed prediction scores of the language modeling head (scores for each vocabulary token before SoftMax)
at each generation step. `(max_length-1,)`-shaped tuple of `tf.Tensor` with each tensor of shape
`(batch_size*num_return_sequences, config.vocab_size)`).
encoder_attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer of the decoder) of shape
- `(batch_size*num_return_sequences, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer of the decoder) of shape `(batch_size*num_return_sequences,
+ num_heads, sequence_length, sequence_length)`.
encoder_hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size*num_return_sequences, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size*num_return_sequences, sequence_length, hidden_size)`.
decoder_attentions (`tuple(tuple(tf.Tensor))`, *optional*, returned when `output_attentions=True` is passed or `config.output_attentions=True`):
Tuple (one element for each generated token) of tuples (one element for each layer of the decoder) of
`tf.Tensor` of shape `(batch_size*num_return_sequences, num_heads, generated_length, sequence_length)`.
@@ -173,8 +174,8 @@ class TFBeamSearchDecoderOnlyOutput(ModelOutput):
Args:
sequences (`tf.Tensor` of shape `(batch_size*num_return_sequences, sequence_length)`):
- The generated sequences. The second dimension (sequence_length) is either equal to `max_length` or
- shorter if all batches finished early due to the `eos_token_id`.
+ The generated sequences. The second dimension (sequence_length) is either equal to `max_length` or shorter
+ if all batches finished early due to the `eos_token_id`.
sequences_scores (`tf.Tensor` of shape `(batch_size*num_return_sequences)`, *optional*, returned when `output_scores=True` is passed or when `config.output_scores=True`):
Final beam scores of the generated `sequences`.
scores (`tuple(tf.Tensor)` *optional*, returned when `output_scores=True` is passed or when `config.output_scores=True`):
@@ -206,24 +207,26 @@ class TFBeamSearchEncoderDecoderOutput(ModelOutput):
Args:
sequences (`tf.Tensor` of shape `(batch_size*num_return_sequences, sequence_length)`):
- The generated sequences. The second dimension (sequence_length) is either equal to `max_length` or
- shorter if all batches finished early due to the `eos_token_id`.
+ The generated sequences. The second dimension (sequence_length) is either equal to `max_length` or shorter
+ if all batches finished early due to the `eos_token_id`.
sequences_scores (`tf.Tensor` of shape `(batch_size*num_return_sequences)`, *optional*, returned when `output_scores=True` is passed or when `config.output_scores=True`):
Final beam scores of the generated `sequences`.
scores (`tuple(tf.Tensor)` *optional*, returned when `output_scores=True` is passed or when `config.output_scores=True`):
Processed beam scores for each vocabulary token at each generation step. Beam scores consisting of log
softmax scores for each vocabulary token and sum of log softmax of previously generated tokens in this beam
- . `(max_length-1,)`-shaped tuple of `tf.Tensor` with each tensor of shape
- `(batch_size*num_beams, config.vocab_size)`).
+ . `(max_length-1,)`-shaped tuple of `tf.Tensor` with each tensor of shape `(batch_size*num_beams,
+ config.vocab_size)`).
attentions (`tuple(tuple(tf.Tensor))`, *optional*, returned when `output_attentions=True` is passed or `config.output_attentions=True`):
encoder_attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer of the decoder) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer of the decoder) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
encoder_hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size*num_beams*num_return_sequences, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size*num_beams*num_return_sequences, sequence_length, hidden_size)`.
decoder_attentions (`tuple(tuple(tf.Tensor))`, *optional*, returned when `output_attentions=True` is passed or `config.output_attentions=True`):
Tuple (one element for each generated token) of tuples (one element for each layer of the decoder) of
- `tf.Tensor` of shape `(batch_size*num_beams*num_return_sequences, num_heads, generated_length, sequence_length)`.
+ `tf.Tensor` of shape `(batch_size*num_beams*num_return_sequences, num_heads, generated_length,
+ sequence_length)`.
cross_attentions (`tuple(tuple(tf.Tensor))`, *optional*, returned when `output_attentions=True` is passed or `config.output_attentions=True`):
Tuple (one element for each generated token) of tuples (one element for each layer of the decoder) of
`tf.Tensor` of shape `(batch_size, num_heads, generated_length, sequence_length)`.
@@ -249,8 +252,8 @@ class TFBeamSampleDecoderOnlyOutput(ModelOutput):
Args:
sequences (`tf.Tensor` of shape `(batch_size*num_return_sequences, sequence_length)`):
- The generated sequences. The second dimension (sequence_length) is either equal to `max_length` or
- shorter if all batches finished early due to the `eos_token_id`.
+ The generated sequences. The second dimension (sequence_length) is either equal to `max_length` or shorter
+ if all batches finished early due to the `eos_token_id`.
sequences_scores (`tf.Tensor` of shape `(batch_size * num_return_sequence)`, *optional*, returned when `output_scores=True` is passed or when `config.output_scores=True`):
Final beam scores of the generated `sequences`.
scores (`tuple(tf.Tensor)` *optional*, returned when `output_scores=True` is passed or when `config.output_scores=True`):
@@ -282,20 +285,21 @@ class TFBeamSampleEncoderDecoderOutput(ModelOutput):
Args:
sequences (`tf.Tensor` of shape `(batch_size*num_beams, sequence_length)`):
- The generated sequences. The second dimension (sequence_length) is either equal to `max_length` or
- shorter if all batches finished early due to the `eos_token_id`.
+ The generated sequences. The second dimension (sequence_length) is either equal to `max_length` or shorter
+ if all batches finished early due to the `eos_token_id`.
sequences_scores (`tf.Tensor` of shape `(batch_size * num_return_sequence)`, *optional*, returned when `output_scores=True` is passed or when `config.output_scores=True`):
Final beam scores of the generated `sequences`.
scores (`tuple(tf.Tensor)` *optional*, returned when `output_scores=True` is passed or when `config.output_scores=True`):
Processed beam scores for each vocabulary token at each generation step. Beam scores consisting of log
softmax scores for each vocabulary token and sum of log softmax of previously generated tokens in this beam
- . `(max_length-1,)`-shaped tuple of `tf.Tensor` with each tensor of shape
- `(batch_size*num_beams, config.vocab_size)`).
+ . `(max_length-1,)`-shaped tuple of `tf.Tensor` with each tensor of shape `(batch_size*num_beams,
+ config.vocab_size)`).
encoder_attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer of the decoder) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer of the decoder) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
encoder_hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size*num_beams, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size*num_beams, sequence_length, hidden_size)`.
decoder_attentions (`tuple(tuple(tf.Tensor))`, *optional*, returned when `output_attentions=True` is passed or `config.output_attentions=True`):
Tuple (one element for each generated token) of tuples (one element for each layer of the decoder) of
`tf.Tensor` of shape `(batch_size*num_beams, num_heads, generated_length, sequence_length)`.
@@ -325,14 +329,12 @@ TFBeamSampleOutput = Union[TFBeamSampleEncoderDecoderOutput, TFBeamSampleDecoder
class TFGenerationMixin:
"""
- A class containing all of the functions supporting generation, to be used as a mixin in
- [`TFPreTrainedModel`].
+ A class containing all of the functions supporting generation, to be used as a mixin in [`TFPreTrainedModel`].
"""
def prepare_inputs_for_generation(self, inputs, **kwargs):
"""
- Implement in subclasses of [`TFPreTrainedModel`] for custom behavior to prepare inputs in
- the generate method.
+ Implement in subclasses of [`TFPreTrainedModel`] for custom behavior to prepare inputs in the generate method.
"""
return {"input_ids": inputs}
@@ -379,13 +381,15 @@ class TFGenerationMixin:
Generates sequences for models with a language modeling head. The method currently supports greedy decoding,
beam-search decoding, sampling with temperature, sampling with top-k or nucleus sampling.
- Adapted in part from [Facebook's XLM beam search code](https://github.com/facebookresearch/XLM/blob/9e6f6814d17be4fe5b15f2e6c43eb2b2d76daeb4/src/model/transformer.py#L529).
+ Adapted in part from [Facebook's XLM beam search
+ code](https://github.com/facebookresearch/XLM/blob/9e6f6814d17be4fe5b15f2e6c43eb2b2d76daeb4/src/model/transformer.py#L529).
- Apart from `input_ids` and `attention_mask`, all the arguments below will default to the value of the
- attribute of the same name inside the [`PretrainedConfig`] of the model. The default values
- indicated are the default values of those config.
+ Apart from `input_ids` and `attention_mask`, all the arguments below will default to the value of the attribute
+ of the same name inside the [`PretrainedConfig`] of the model. The default values indicated are the default
+ values of those config.
- Most of these parameters are explained in more detail in [this blog post](https://huggingface.co/blog/how-to-generate).
+ Most of these parameters are explained in more detail in [this blog
+ post](https://huggingface.co/blog/how-to-generate).
Parameters:
@@ -407,10 +411,11 @@ class TFGenerationMixin:
top_k (`int`, *optional*, defaults to 50):
The number of highest probability vocabulary tokens to keep for top-k-filtering.
top_p (`float`, *optional*, defaults to 1.0):
- If set to float < 1, only the most probable tokens with probabilities that add up to `top_p` or
- higher are kept for generation.
+ If set to float < 1, only the most probable tokens with probabilities that add up to `top_p` or higher
+ are kept for generation.
repetition_penalty (`float`, *optional*, defaults to 1.0):
- The parameter for repetition penalty. 1.0 means no penalty. See [this paper](https://arxiv.org/pdf/1909.05858.pdf) for more details.
+ The parameter for repetition penalty. 1.0 means no penalty. See [this
+ paper](https://arxiv.org/pdf/1909.05858.pdf) for more details.
pad_token_id (`int`, *optional*):
The id of the *padding* token.
bos_token_id (`int`, *optional*):
@@ -430,8 +435,8 @@ class TFGenerationMixin:
num_return_sequences(`int`, *optional*, defaults to 1):
The number of independently computed returned sequences for each element in the batch.
attention_mask (`tf.Tensor` of `dtype=tf.int32` and shape `(batch_size, sequence_length)`, *optional*):
- Mask to avoid performing attention on padding token indices. Mask values are in `[0, 1]`, 1 for
- tokens that are not masked, and 0 for masked tokens.
+ Mask to avoid performing attention on padding token indices. Mask values are in `[0, 1]`, 1 for tokens
+ that are not masked, and 0 for masked tokens.
If not provided, will default to a tensor the same shape as `input_ids` that masks the pad token.
@@ -452,21 +457,20 @@ class TFGenerationMixin:
return_dict_in_generate (`bool`, *optional*, defaults to *False*):
Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple.
forced_bos_token_id (`int`, *optional*):
- The id of the token to force as the first generated token after the `decoder_start_token_id`.
- Useful for multilingual models like [mBART](../model_doc/mbart) where the first generated token
- needs to be the target language token.
+ The id of the token to force as the first generated token after the `decoder_start_token_id`. Useful
+ for multilingual models like [mBART](../model_doc/mbart) where the first generated token needs to be
+ the target language token.
forced_eos_token_id (`int`, *optional*):
The id of the token to force as the last generated token when `max_length` is reached.
model_specific_kwargs:
Additional model specific kwargs will be forwarded to the `forward` function of the model.
Return:
- [`~file_utils.ModelOutput`] or `tf.Tensor`: A
- [`~file_utils.ModelOutput`] (if `return_dict_in_generate=True` or when
- `config.return_dict_in_generate=True`) or a `tf.Tensor`.
+ [`~file_utils.ModelOutput`] or `tf.Tensor`: A [`~file_utils.ModelOutput`] (if
+ `return_dict_in_generate=True` or when `config.return_dict_in_generate=True`) or a `tf.Tensor`.
- If the model is *not* an encoder-decoder model (`model.config.is_encoder_decoder=False`), the
- possible [`~file_utils.ModelOutput`] types are:
+ If the model is *not* an encoder-decoder model (`model.config.is_encoder_decoder=False`), the possible
+ [`~file_utils.ModelOutput`] types are:
- [`~generation_utils.TFGreedySearchDecoderOnlyOutput`],
- [`~generation_utils.TFSampleDecoderOnlyOutput`],
@@ -1438,8 +1442,7 @@ class TFGenerationMixin:
self, logits, cur_len, max_length, forced_bos_token_id, forced_eos_token_id, **kwargs
):
"""
- Implement in subclasses of [`PreTrainedModel`] for custom behavior to adjust the logits in
- the generate method.
+ Implement in subclasses of [`PreTrainedModel`] for custom behavior to adjust the logits in the generate method.
"""
vocab_size = getattr(self.config, "vocab_size", None)
if vocab_size is None and self.config.is_encoder_decoder:
diff --git a/src/transformers/generation_utils.py b/src/transformers/generation_utils.py
index 58d13c2e27..5ef9cf50ac 100644
--- a/src/transformers/generation_utils.py
+++ b/src/transformers/generation_utils.py
@@ -62,12 +62,12 @@ class GreedySearchDecoderOnlyOutput(ModelOutput):
Args:
sequences (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
- The generated sequences. The second dimension (sequence_length) is either equal to `max_length` or
- shorter if all batches finished early due to the `eos_token_id`.
+ The generated sequences. The second dimension (sequence_length) is either equal to `max_length` or shorter
+ if all batches finished early due to the `eos_token_id`.
scores (`tuple(torch.FloatTensor)` *optional*, returned when `output_scores=True` is passed or when `config.output_scores=True`):
Processed prediction scores of the language modeling head (scores for each vocabulary token before SoftMax)
- at each generation step. `(max_length-input_ids.shape[-1],)`-shaped tuple of `torch.FloatTensor`
- with each tensor of shape `(batch_size, config.vocab_size)`).
+ at each generation step. `(max_length-input_ids.shape[-1],)`-shaped tuple of `torch.FloatTensor` with each
+ tensor of shape `(batch_size, config.vocab_size)`).
attentions (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `output_attentions=True` is passed or `config.output_attentions=True`):
Tuple (one element for each generated token) of tuples (one element for each layer of the decoder) of
`torch.FloatTensor` of shape `(batch_size, num_heads, generated_length, sequence_length)`.
@@ -92,17 +92,18 @@ class GreedySearchEncoderDecoderOutput(ModelOutput):
Args:
sequences (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
- The generated sequences. The second dimension (sequence_length) is either equal to `max_length` or
- shorter if all batches finished early due to the `eos_token_id`.
+ The generated sequences. The second dimension (sequence_length) is either equal to `max_length` or shorter
+ if all batches finished early due to the `eos_token_id`.
scores (`tuple(torch.FloatTensor)` *optional*, returned when `output_scores=True` is passed or when `config.output_scores=True`):
Processed prediction scores of the language modeling head (scores for each vocabulary token before SoftMax)
- at each generation step. `(max_length-1,)`-shaped tuple of `torch.FloatTensor` with each tensor
- of shape `(batch_size, config.vocab_size)`).
+ at each generation step. `(max_length-1,)`-shaped tuple of `torch.FloatTensor` with each tensor of shape
+ `(batch_size, config.vocab_size)`).
encoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer of the decoder) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer of the decoder) of shape `(batch_size, num_heads,
+ sequence_length, sequence_length)`.
encoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
decoder_attentions (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `output_attentions=True` is passed or `config.output_attentions=True`):
Tuple (one element for each generated token) of tuples (one element for each layer of the decoder) of
`torch.FloatTensor` of shape `(batch_size, num_heads, generated_length, sequence_length)`.
@@ -131,15 +132,16 @@ class SampleDecoderOnlyOutput(ModelOutput):
Args:
sequences (`torch.LongTensor` of shape `(batch_size*num_return_sequences, sequence_length)`):
- The generated sequences. The second dimension (sequence_length) is either equal to `max_length` or
- shorter if all batches finished early due to the `eos_token_id`.
+ The generated sequences. The second dimension (sequence_length) is either equal to `max_length` or shorter
+ if all batches finished early due to the `eos_token_id`.
scores (`tuple(torch.FloatTensor)` *optional*, returned when `output_scores=True` is passed or when `config.output_scores=True`):
Processed prediction scores of the language modeling head (scores for each vocabulary token before SoftMax)
- at each generation step. `(max_length-input_ids.shape[-1],)`-shaped tuple of `torch.FloatTensor`
- with each tensor of shape `(batch_size*num_return_sequences, config.vocab_size)`).
+ at each generation step. `(max_length-input_ids.shape[-1],)`-shaped tuple of `torch.FloatTensor` with each
+ tensor of shape `(batch_size*num_return_sequences, config.vocab_size)`).
attentions (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `output_attentions=True` is passed or `config.output_attentions=True`):
Tuple (one element for each generated token) of tuples (one element for each layer of the decoder) of
- `torch.FloatTensor` of shape `(num_return_sequences*batch_size, num_heads, generated_length, sequence_length)`.
+ `torch.FloatTensor` of shape `(num_return_sequences*batch_size, num_heads, generated_length,
+ sequence_length)`.
hidden_states (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
Tuple (one element for each generated token) of tuples (one element for each layer of the decoder) of
`torch.FloatTensor` of shape `(num_return_sequences*batch_size, generated_length, hidden_size)`.
@@ -161,21 +163,22 @@ class SampleEncoderDecoderOutput(ModelOutput):
Args:
sequences (`torch.LongTensor` of shape `(batch_size*num_return_sequences, sequence_length)`):
- The generated sequences. The second dimension (sequence_length) is either equal to `max_length` or
- shorter if all batches finished early due to the `eos_token_id`.
+ The generated sequences. The second dimension (sequence_length) is either equal to `max_length` or shorter
+ if all batches finished early due to the `eos_token_id`.
scores (`tuple(torch.FloatTensor)` *optional*, returned when `output_scores=True` is passed or when `config.output_scores=True`):
Processed prediction scores of the language modeling head (scores for each vocabulary token before SoftMax)
- at each generation step. `(max_length-1,)`-shaped tuple of `torch.FloatTensor` with each tensor
- of shape `(batch_size*num_return_sequences, config.vocab_size)`).
+ at each generation step. `(max_length-1,)`-shaped tuple of `torch.FloatTensor` with each tensor of shape
+ `(batch_size*num_return_sequences, config.vocab_size)`).
encoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or `config.output_attentions=True`):
Tuple of `torch.FloatTensor` (one for each layer of the decoder) of shape
`(batch_size*num_return_sequences, num_heads, sequence_length, sequence_length)`.
encoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size*num_return_sequences, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size*num_return_sequences, sequence_length, hidden_size)`.
decoder_attentions (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `output_attentions=True` is passed or `config.output_attentions=True`):
Tuple (one element for each generated token) of tuples (one element for each layer of the decoder) of
- `torch.FloatTensor` of shape `(batch_size*num_return_sequences, num_heads, generated_length, sequence_length)`.
+ `torch.FloatTensor` of shape `(batch_size*num_return_sequences, num_heads, generated_length,
+ sequence_length)`.
cross_attentions (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `output_attentions=True` is passed or `config.output_attentions=True`):
Tuple (one element for each generated token) of tuples (one element for each layer of the decoder) of
`torch.FloatTensor` of shape `(batch_size, num_heads, generated_length, sequence_length)`.
@@ -200,15 +203,15 @@ class BeamSearchDecoderOnlyOutput(ModelOutput):
Args:
sequences (`torch.LongTensor` of shape `(batch_size*num_return_sequences, sequence_length)`):
- The generated sequences. The second dimension (sequence_length) is either equal to `max_length` or
- shorter if all batches finished early due to the `eos_token_id`.
+ The generated sequences. The second dimension (sequence_length) is either equal to `max_length` or shorter
+ if all batches finished early due to the `eos_token_id`.
sequences_scores (`torch.FloatTensor` of shape `(batch_size*num_return_sequences)`, *optional*, returned when `output_scores=True` is passed or when `config.output_scores=True`):
Final beam scores of the generated `sequences`.
scores (`tuple(torch.FloatTensor)` *optional*, returned when `output_scores=True` is passed or when `config.output_scores=True`):
Processed beam scores for each vocabulary token at each generation step. Beam scores consisting of log
softmax scores for each vocabulary token and sum of log softmax of previously generated tokens in this beam
- . `(max_length-input_ids.shape[-1],)`-shaped tuple of `torch.FloatTensor` with each tensor of
- shape `(batch_size*num_beams*num_return_sequences, config.vocab_size)`).
+ . `(max_length-input_ids.shape[-1],)`-shaped tuple of `torch.FloatTensor` with each tensor of shape
+ `(batch_size*num_beams*num_return_sequences, config.vocab_size)`).
attentions (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `output_attentions=True` is passed or `config.output_attentions=True`):
Tuple (one element for each generated token) of tuples (one element for each layer of the decoder) of
`torch.FloatTensor` of shape `(batch_size*num_beams, num_heads, generated_length, sequence_length)`.
@@ -233,24 +236,26 @@ class BeamSearchEncoderDecoderOutput(ModelOutput):
Args:
sequences (`torch.LongTensor` of shape `(batch_size*num_return_sequences, sequence_length)`):
- The generated sequences. The second dimension (sequence_length) is either equal to `max_length` or
- shorter if all batches finished early due to the `eos_token_id`.
+ The generated sequences. The second dimension (sequence_length) is either equal to `max_length` or shorter
+ if all batches finished early due to the `eos_token_id`.
sequences_scores (`torch.FloatTensor` of shape `(batch_size*num_return_sequences)`, *optional*, returned when `output_scores=True` is passed or when `config.output_scores=True`):
Final beam scores of the generated `sequences`.
scores (`tuple(torch.FloatTensor)` *optional*, returned when `output_scores=True` is passed or when `config.output_scores=True`):
Processed beam scores for each vocabulary token at each generation step. Beam scores consisting of log
softmax scores for each vocabulary token and sum of log softmax of previously generated tokens in this beam
- . `(max_length-1,)`-shaped tuple of `torch.FloatTensor` with each tensor of shape
- `(batch_size*num_beams, config.vocab_size)`).
+ . `(max_length-1,)`-shaped tuple of `torch.FloatTensor` with each tensor of shape `(batch_size*num_beams,
+ config.vocab_size)`).
attentions (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `output_attentions=True` is passed or `config.output_attentions=True`):
encoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer of the decoder) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer of the decoder) of shape `(batch_size, num_heads,
+ sequence_length, sequence_length)`.
encoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size*num_beams*num_return_sequences, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size*num_beams*num_return_sequences, sequence_length, hidden_size)`.
decoder_attentions (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `output_attentions=True` is passed or `config.output_attentions=True`):
Tuple (one element for each generated token) of tuples (one element for each layer of the decoder) of
- `torch.FloatTensor` of shape `(batch_size*num_beams*num_return_sequences, num_heads, generated_length, sequence_length)`.
+ `torch.FloatTensor` of shape `(batch_size*num_beams*num_return_sequences, num_heads, generated_length,
+ sequence_length)`.
cross_attentions (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `output_attentions=True` is passed or `config.output_attentions=True`):
Tuple (one element for each generated token) of tuples (one element for each layer of the decoder) of
`torch.FloatTensor` of shape `(batch_size, num_heads, generated_length, sequence_length)`.
@@ -276,15 +281,15 @@ class BeamSampleDecoderOnlyOutput(ModelOutput):
Args:
sequences (`torch.LongTensor` of shape `(batch_size*num_return_sequences, sequence_length)`):
- The generated sequences. The second dimension (sequence_length) is either equal to `max_length` or
- shorter if all batches finished early due to the `eos_token_id`.
+ The generated sequences. The second dimension (sequence_length) is either equal to `max_length` or shorter
+ if all batches finished early due to the `eos_token_id`.
sequences_scores (`torch.FloatTensor` of shape `(batch_size * num_return_sequence)`, *optional*, returned when `output_scores=True` is passed or when `config.output_scores=True`):
Final beam scores of the generated `sequences`.
scores (`tuple(torch.FloatTensor)` *optional*, returned when `output_scores=True` is passed or when `config.output_scores=True`):
Processed beam scores for each vocabulary token at each generation step. Beam scores consisting of log
softmax scores for each vocabulary token and sum of log softmax of previously generated tokens in this beam
- . `(max_length-input_ids.shape[-1],)`-shaped tuple of `torch.FloatTensor` with each tensor of
- shape `(batch_size*num_beams*num_return_sequences, config.vocab_size)`).
+ . `(max_length-input_ids.shape[-1],)`-shaped tuple of `torch.FloatTensor` with each tensor of shape
+ `(batch_size*num_beams*num_return_sequences, config.vocab_size)`).
attentions (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `output_attentions=True` is passed or `config.output_attentions=True`):
Tuple (one element for each generated token) of tuples (one element for each layer of the decoder) of
`torch.FloatTensor` of shape `(batch_size*num_beams, num_heads, generated_length, sequence_length)`.
@@ -309,20 +314,21 @@ class BeamSampleEncoderDecoderOutput(ModelOutput):
Args:
sequences (`torch.LongTensor` of shape `(batch_size*num_beams, sequence_length)`):
- The generated sequences. The second dimension (sequence_length) is either equal to `max_length` or
- shorter if all batches finished early due to the `eos_token_id`.
+ The generated sequences. The second dimension (sequence_length) is either equal to `max_length` or shorter
+ if all batches finished early due to the `eos_token_id`.
sequences_scores (`torch.FloatTensor` of shape `(batch_size * num_return_sequence)`, *optional*, returned when `output_scores=True` is passed or when `config.output_scores=True`):
Final beam scores of the generated `sequences`.
scores (`tuple(torch.FloatTensor)` *optional*, returned when `output_scores=True` is passed or when `config.output_scores=True`):
Processed beam scores for each vocabulary token at each generation step. Beam scores consisting of log
softmax scores for each vocabulary token and sum of log softmax of previously generated tokens in this beam
- . `(max_length-1,)`-shaped tuple of `torch.FloatTensor` with each tensor of shape
- `(batch_size*num_beams, config.vocab_size)`).
+ . `(max_length-1,)`-shaped tuple of `torch.FloatTensor` with each tensor of shape `(batch_size*num_beams,
+ config.vocab_size)`).
encoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer of the decoder) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer of the decoder) of shape `(batch_size, num_heads,
+ sequence_length, sequence_length)`.
encoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size*num_beams, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size*num_beams, sequence_length, hidden_size)`.
decoder_attentions (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `output_attentions=True` is passed or `config.output_attentions=True`):
Tuple (one element for each generated token) of tuples (one element for each layer of the decoder) of
`torch.FloatTensor` of shape `(batch_size*num_beams, num_heads, generated_length, sequence_length)`.
@@ -352,8 +358,7 @@ BeamSampleOutput = Union[BeamSampleEncoderDecoderOutput, BeamSampleDecoderOnlyOu
class GenerationMixin:
"""
- A class containing all of the functions supporting generation, to be used as a mixin in
- [`PreTrainedModel`].
+ A class containing all of the functions supporting generation, to be used as a mixin in [`PreTrainedModel`].
"""
def _prepare_model_inputs(
@@ -412,8 +417,8 @@ class GenerationMixin:
self, inputs: Optional[torch.Tensor], name: str, model_kwargs: Dict[str, torch.Tensor]
) -> torch.Tensor:
"""
- If `inputs` is None and `name` is in both forward function and keyword
- arguments, then inputs can be retrieved from name
+ If `inputs` is None and `name` is in both forward function and keyword arguments, then inputs can be retrieved
+ from name
"""
can_retrieve_inputs = model_kwargs.get(name, None) is not None and name in set(
inspect.signature(self.forward).parameters.keys()
@@ -426,15 +431,13 @@ class GenerationMixin:
def prepare_inputs_for_generation(self, input_ids: torch.LongTensor, **kwargs) -> Dict[str, Any]:
"""
- Implement in subclasses of [`PreTrainedModel`] for custom behavior to prepare inputs in the
- generate method.
+ Implement in subclasses of [`PreTrainedModel`] for custom behavior to prepare inputs in the generate method.
"""
return {"input_ids": input_ids}
def adjust_logits_during_generation(self, logits: torch.FloatTensor, **kwargs) -> torch.FloatTensor:
"""
- Implement in subclasses of [`PreTrainedModel`] for custom behavior to adjust the logits in
- the generate method.
+ Implement in subclasses of [`PreTrainedModel`] for custom behavior to adjust the logits in the generate method.
"""
return logits
@@ -603,8 +606,8 @@ class GenerationMixin:
self, top_k: int = None, top_p: float = None, temperature: float = None, num_beams: int = None
) -> LogitsProcessorList:
"""
- This class returns a [`LogitsProcessorList`] list object that contains all relevant
- [`LogitsWarper`] instances used for multinomial sampling.
+ This class returns a [`LogitsProcessorList`] list object that contains all relevant [`LogitsWarper`] instances
+ used for multinomial sampling.
"""
# init warp parameters
@@ -644,8 +647,8 @@ class GenerationMixin:
logits_processor: Optional[LogitsProcessorList],
) -> LogitsProcessorList:
"""
- This class returns a [`LogitsProcessorList`] list object that contains all relevant
- [`LogitsProcessor`] instances used to modify the scores of the language model head.
+ This class returns a [`LogitsProcessorList`] list object that contains all relevant [`LogitsProcessor`]
+ instances used to modify the scores of the language model head.
"""
processors = LogitsProcessorList()
@@ -784,19 +787,20 @@ class GenerationMixin:
Generates sequences for models with a language modeling head. The method currently supports greedy decoding,
multinomial sampling, beam-search decoding, and beam-search multinomial sampling.
- Apart from `inputs`, all the arguments below will default to the value of the attribute of the same name
- inside the [`PretrainedConfig`] of the model. The default values indicated are the default
- values of those config.
+ Apart from `inputs`, all the arguments below will default to the value of the attribute of the same name inside
+ the [`PretrainedConfig`] of the model. The default values indicated are the default values of those config.
- Most of these parameters are explained in more detail in [this blog post](https://huggingface.co/blog/how-to-generate).
+ Most of these parameters are explained in more detail in [this blog
+ post](https://huggingface.co/blog/how-to-generate).
Parameters:
- inputs (`torch.Tensor` of shape `(batch_size, sequence_length)`, `(batch_size, sequence_length, feature_dim)` or `(batch_size, num_channels, height, width)`, *optional*):
+ inputs (`torch.Tensor` of shape `(batch_size, sequence_length)`, `(batch_size, sequence_length,
+ feature_dim)` or `(batch_size, num_channels, height, width)`, *optional*):
The sequence used as a prompt for the generation or as model inputs to the encoder. If `None` the
- method initializes it with `bos_token_id` and a batch size of 1. For decoder-only models
- `inputs` should of in the format of `input_ids`. For encoder-decoder models *inputs* can
- represent any of `input_ids`, `input_values`, `input_features`, or `pixel_values`.
+ method initializes it with `bos_token_id` and a batch size of 1. For decoder-only models `inputs`
+ should of in the format of `input_ids`. For encoder-decoder models *inputs* can represent any of
+ `input_ids`, `input_values`, `input_features`, or `pixel_values`.
max_length (`int`, *optional*, defaults to `model.config.max_length`):
The maximum length of the sequence to be generated.
max_new_tokens (`int`, *optional*, defaults to None):
@@ -815,10 +819,11 @@ class GenerationMixin:
top_k (`int`, *optional*, defaults to 50):
The number of highest probability vocabulary tokens to keep for top-k-filtering.
top_p (`float`, *optional*, defaults to 1.0):
- If set to float < 1, only the most probable tokens with probabilities that add up to `top_p` or
- higher are kept for generation.
+ If set to float < 1, only the most probable tokens with probabilities that add up to `top_p` or higher
+ are kept for generation.
repetition_penalty (`float`, *optional*, defaults to 1.0):
- The parameter for repetition penalty. 1.0 means no penalty. See [this paper](https://arxiv.org/pdf/1909.05858.pdf) for more details.
+ The parameter for repetition penalty. 1.0 means no penalty. See [this
+ paper](https://arxiv.org/pdf/1909.05858.pdf) for more details.
pad_token_id (`int`, *optional*):
The id of the *padding* token.
bos_token_id (`int`, *optional*):
@@ -843,9 +848,9 @@ class GenerationMixin:
The maximum amount of time you allow the computation to run for in seconds. generation will still
finish the current pass after allocated time has been passed.
attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Mask to avoid performing attention on padding token indices. Mask values are in `[0, 1]`, 1 for
- tokens that are not masked, and 0 for masked tokens. If not provided, will default to a tensor the same
- shape as `input_ids` that masks the pad token. [What are attention masks?](../glossary#attention-mask)
+ Mask to avoid performing attention on padding token indices. Mask values are in `[0, 1]`, 1 for tokens
+ that are not masked, and 0 for masked tokens. If not provided, will default to a tensor the same shape
+ as `input_ids` that masks the pad token. [What are attention masks?](../glossary#attention-mask)
decoder_start_token_id (`int`, *optional*):
If an encoder-decoder model starts decoding with a different token than *bos*, the id of that token.
use_cache: (`bool`, *optional*, defaults to `True`):
@@ -861,10 +866,10 @@ class GenerationMixin:
prefix_allowed_tokens_fn: (`Callable[[int, torch.Tensor], List[int]]`, *optional*):
If provided, this function constraints the beam search to allowed tokens only at each step. If not
provided no constraint is applied. This function takes 2 arguments: the batch ID `batch_id` and
- `input_ids`. It has to return a list with the allowed tokens for the next generation step
- conditioned on the batch ID `batch_id` and the previously generated tokens `inputs_ids`. This
- argument is useful for constrained generation conditioned on the prefix, as described in
- [Autoregressive Entity Retrieval](https://arxiv.org/abs/2010.00904).
+ `input_ids`. It has to return a list with the allowed tokens for the next generation step conditioned
+ on the batch ID `batch_id` and the previously generated tokens `inputs_ids`. This argument is useful
+ for constrained generation conditioned on the prefix, as described in [Autoregressive Entity
+ Retrieval](https://arxiv.org/abs/2010.00904).
logits_processor (`LogitsProcessorList`, *optional*):
Custom logits processors that complement the default logits processors built from arguments and a
model's config. If a logit processor is passed that is already created with the arguments or a model's
@@ -884,9 +889,9 @@ class GenerationMixin:
return_dict_in_generate (`bool`, *optional*, defaults to *False*):
Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple.
forced_bos_token_id (`int`, *optional*):
- The id of the token to force as the first generated token after the `decoder_start_token_id`.
- Useful for multilingual models like [mBART](../model_doc/mbart) where the first generated token
- needs to be the target language token.
+ The id of the token to force as the first generated token after the `decoder_start_token_id`. Useful
+ for multilingual models like [mBART](../model_doc/mbart) where the first generated token needs to be
+ the target language token.
forced_eos_token_id (`int`, *optional*):
The id of the token to force as the last generated token when `max_length` is reached.
remove_invalid_values (`bool`, *optional*):
@@ -896,17 +901,16 @@ class GenerationMixin:
Whether to continue running the while loop until max_length (needed for ZeRO stage 3)
model_kwargs:
- Additional model specific kwargs will be forwarded to the `forward` function of the model. If the
- model is an encoder-decoder model, encoder specific kwargs should not be prefixed and decoder specific
- kwargs should be prefixed with *decoder_*.
+ Additional model specific kwargs will be forwarded to the `forward` function of the model. If the model
+ is an encoder-decoder model, encoder specific kwargs should not be prefixed and decoder specific kwargs
+ should be prefixed with *decoder_*.
Return:
- [`~file_utils.ModelOutput`] or `torch.LongTensor`: A
- [`~file_utils.ModelOutput`] (if `return_dict_in_generate=True` or when
- `config.return_dict_in_generate=True`) or a `torch.FloatTensor`.
+ [`~file_utils.ModelOutput`] or `torch.LongTensor`: A [`~file_utils.ModelOutput`] (if
+ `return_dict_in_generate=True` or when `config.return_dict_in_generate=True`) or a `torch.FloatTensor`.
- If the model is *not* an encoder-decoder model (`model.config.is_encoder_decoder=False`), the
- possible [`~file_utils.ModelOutput`] types are:
+ If the model is *not* an encoder-decoder model (`model.config.is_encoder_decoder=False`), the possible
+ [`~file_utils.ModelOutput`] types are:
- [`~generation_utils.GreedySearchDecoderOnlyOutput`],
- [`~generation_utils.SampleDecoderOnlyOutput`],
@@ -1285,16 +1289,15 @@ class GenerationMixin:
input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
The sequence used as a prompt for the generation.
logits_processor (`LogitsProcessorList`, *optional*):
- An instance of [`LogitsProcessorList`]. List of instances of class derived from
- [`LogitsProcessor`] used to modify the prediction scores of the language modeling
- head applied at each generation step.
+ An instance of [`LogitsProcessorList`]. List of instances of class derived from [`LogitsProcessor`]
+ used to modify the prediction scores of the language modeling head applied at each generation step.
stopping_criteria (`StoppingCriteriaList`, *optional*):
- An instance of [`StoppingCriteriaList`]. List of instances of class derived from
- [`StoppingCriteria`] used to tell if the generation loop should stop.
+ An instance of [`StoppingCriteriaList`]. List of instances of class derived from [`StoppingCriteria`]
+ used to tell if the generation loop should stop.
max_length (`int`, *optional*, defaults to 20):
- **DEPRECATED**. Use `logits_processor` or `stopping_criteria` directly to cap the number of
- generated tokens. The maximum length of the sequence to be generated.
+ **DEPRECATED**. Use `logits_processor` or `stopping_criteria` directly to cap the number of generated
+ tokens. The maximum length of the sequence to be generated.
pad_token_id (`int`, *optional*):
The id of the *padding* token.
eos_token_id (`int`, *optional*):
@@ -1312,16 +1315,14 @@ class GenerationMixin:
synced_gpus (`bool`, *optional*, defaults to `False`):
Whether to continue running the while loop until max_length (needed for ZeRO stage 3)
model_kwargs:
- Additional model specific keyword arguments will be forwarded to the `forward` function of the
- model. If model is an encoder-decoder model the kwargs should include `encoder_outputs`.
+ Additional model specific keyword arguments will be forwarded to the `forward` function of the model.
+ If model is an encoder-decoder model the kwargs should include `encoder_outputs`.
Return:
- [`~generation_utils.GreedySearchDecoderOnlyOutput`],
- [`~generation_utils.GreedySearchEncoderDecoderOutput`] or obj:*torch.LongTensor*: A
- `torch.LongTensor` containing the generated tokens (default behaviour) or a
- [`~generation_utils.GreedySearchDecoderOnlyOutput`] if
- `model.config.is_encoder_decoder=False` and `return_dict_in_generate=True` or a
- [`~generation_utils.GreedySearchEncoderDecoderOutput`] if
+ [`~generation_utils.GreedySearchDecoderOnlyOutput`], [`~generation_utils.GreedySearchEncoderDecoderOutput`]
+ or obj:*torch.LongTensor*: A `torch.LongTensor` containing the generated tokens (default behaviour) or a
+ [`~generation_utils.GreedySearchDecoderOnlyOutput`] if `model.config.is_encoder_decoder=False` and
+ `return_dict_in_generate=True` or a [`~generation_utils.GreedySearchEncoderDecoderOutput`] if
`model.config.is_encoder_decoder=True`.
Examples:
@@ -1512,19 +1513,18 @@ class GenerationMixin:
input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
The sequence used as a prompt for the generation.
logits_processor (`LogitsProcessorList`, *optional*):
- An instance of [`LogitsProcessorList`]. List of instances of class derived from
- [`LogitsProcessor`] used to modify the prediction scores of the language modeling
- head applied at each generation step.
+ An instance of [`LogitsProcessorList`]. List of instances of class derived from [`LogitsProcessor`]
+ used to modify the prediction scores of the language modeling head applied at each generation step.
stopping_criteria (`StoppingCriteriaList`, *optional*):
- An instance of [`StoppingCriteriaList`]. List of instances of class derived from
- [`StoppingCriteria`] used to tell if the generation loop should stop.
+ An instance of [`StoppingCriteriaList`]. List of instances of class derived from [`StoppingCriteria`]
+ used to tell if the generation loop should stop.
logits_warper (`LogitsProcessorList`, *optional*):
- An instance of [`LogitsProcessorList`]. List of instances of class derived from
- [`LogitsWarper`] used to warp the prediction score distribution of the language
- modeling head applied before multinomial sampling at each generation step.
+ An instance of [`LogitsProcessorList`]. List of instances of class derived from [`LogitsWarper`] used
+ to warp the prediction score distribution of the language modeling head applied before multinomial
+ sampling at each generation step.
max_length (`int`, *optional*, defaults to 20):
- **DEPRECATED**. Use `logits_processor` or `stopping_criteria` directly to cap the number of
- generated tokens. The maximum length of the sequence to be generated.
+ **DEPRECATED**. Use `logits_processor` or `stopping_criteria` directly to cap the number of generated
+ tokens. The maximum length of the sequence to be generated.
pad_token_id (`int`, *optional*):
The id of the *padding* token.
eos_token_id (`int`, *optional*):
@@ -1542,16 +1542,14 @@ class GenerationMixin:
synced_gpus (`bool`, *optional*, defaults to `False`):
Whether to continue running the while loop until max_length (needed for ZeRO stage 3)
model_kwargs:
- Additional model specific kwargs will be forwarded to the `forward` function of the model. If
- model is an encoder-decoder model the kwargs should include `encoder_outputs`.
+ Additional model specific kwargs will be forwarded to the `forward` function of the model. If model is
+ an encoder-decoder model the kwargs should include `encoder_outputs`.
Return:
- [`~generation_utils.SampleDecoderOnlyOutput`],
- [`~generation_utils.SampleEncoderDecoderOutput`] or obj:*torch.LongTensor*: A
- `torch.LongTensor` containing the generated tokens (default behaviour) or a
- [`~generation_utils.SampleDecoderOnlyOutput`] if
- `model.config.is_encoder_decoder=False` and `return_dict_in_generate=True` or a
- [`~generation_utils.SampleEncoderDecoderOutput`] if
+ [`~generation_utils.SampleDecoderOnlyOutput`], [`~generation_utils.SampleEncoderDecoderOutput`] or
+ obj:*torch.LongTensor*: A `torch.LongTensor` containing the generated tokens (default behaviour) or a
+ [`~generation_utils.SampleDecoderOnlyOutput`] if `model.config.is_encoder_decoder=False` and
+ `return_dict_in_generate=True` or a [`~generation_utils.SampleEncoderDecoderOutput`] if
`model.config.is_encoder_decoder=True`.
Examples:
@@ -1754,19 +1752,17 @@ class GenerationMixin:
input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
The sequence used as a prompt for the generation.
beam_scorer (`BeamScorer`):
- An derived instance of [`BeamScorer`] that defines how beam hypotheses are
- constructed, stored and sorted during generation. For more information, the documentation of
- [`BeamScorer`] should be read.
+ An derived instance of [`BeamScorer`] that defines how beam hypotheses are constructed, stored and
+ sorted during generation. For more information, the documentation of [`BeamScorer`] should be read.
logits_processor (`LogitsProcessorList`, *optional*):
- An instance of [`LogitsProcessorList`]. List of instances of class derived from
- [`LogitsProcessor`] used to modify the prediction scores of the language modeling
- head applied at each generation step.
+ An instance of [`LogitsProcessorList`]. List of instances of class derived from [`LogitsProcessor`]
+ used to modify the prediction scores of the language modeling head applied at each generation step.
stopping_criteria (`StoppingCriteriaList`, *optional*):
- An instance of [`StoppingCriteriaList`]. List of instances of class derived from
- [`StoppingCriteria`] used to tell if the generation loop should stop.
+ An instance of [`StoppingCriteriaList`]. List of instances of class derived from [`StoppingCriteria`]
+ used to tell if the generation loop should stop.
max_length (`int`, *optional*, defaults to 20):
- **DEPRECATED**. Use `logits_processor` or `stopping_criteria` directly to cap the number of
- generated tokens. The maximum length of the sequence to be generated.
+ **DEPRECATED**. Use `logits_processor` or `stopping_criteria` directly to cap the number of generated
+ tokens. The maximum length of the sequence to be generated.
pad_token_id (`int`, *optional*):
The id of the *padding* token.
eos_token_id (`int`, *optional*):
@@ -1784,16 +1780,14 @@ class GenerationMixin:
synced_gpus (`bool`, *optional*, defaults to `False`):
Whether to continue running the while loop until max_length (needed for ZeRO stage 3)
model_kwargs:
- Additional model specific kwargs will be forwarded to the `forward` function of the model. If
- model is an encoder-decoder model the kwargs should include `encoder_outputs`.
+ Additional model specific kwargs will be forwarded to the `forward` function of the model. If model is
+ an encoder-decoder model the kwargs should include `encoder_outputs`.
Return:
- [`generation_utilsBeamSearchDecoderOnlyOutput`],
- [`~generation_utils.BeamSearchEncoderDecoderOutput`] or obj:*torch.LongTensor*: A
- `torch.LongTensor` containing the generated tokens (default behaviour) or a
- [`~generation_utils.BeamSearchDecoderOnlyOutput`] if
- `model.config.is_encoder_decoder=False` and `return_dict_in_generate=True` or a
- [`~generation_utils.BeamSearchEncoderDecoderOutput`] if
+ [`generation_utilsBeamSearchDecoderOnlyOutput`], [`~generation_utils.BeamSearchEncoderDecoderOutput`] or
+ obj:*torch.LongTensor*: A `torch.LongTensor` containing the generated tokens (default behaviour) or a
+ [`~generation_utils.BeamSearchDecoderOnlyOutput`] if `model.config.is_encoder_decoder=False` and
+ `return_dict_in_generate=True` or a [`~generation_utils.BeamSearchEncoderDecoderOutput`] if
`model.config.is_encoder_decoder=True`.
@@ -2048,23 +2042,21 @@ class GenerationMixin:
input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
The sequence used as a prompt for the generation.
beam_scorer (`BeamScorer`):
- A derived instance of [`BeamScorer`] that defines how beam hypotheses are
- constructed, stored and sorted during generation. For more information, the documentation of
- [`BeamScorer`] should be read.
+ A derived instance of [`BeamScorer`] that defines how beam hypotheses are constructed, stored and
+ sorted during generation. For more information, the documentation of [`BeamScorer`] should be read.
logits_processor (`LogitsProcessorList`, *optional*):
- An instance of [`LogitsProcessorList`]. List of instances of class derived from
- [`LogitsProcessor`] used to modify the prediction scores of the language modeling
- head applied at each generation step.
+ An instance of [`LogitsProcessorList`]. List of instances of class derived from [`LogitsProcessor`]
+ used to modify the prediction scores of the language modeling head applied at each generation step.
stopping_criteria (`StoppingCriteriaList`, *optional*):
- An instance of [`StoppingCriteriaList`]. List of instances of class derived from
- [`StoppingCriteria`] used to tell if the generation loop should stop.
+ An instance of [`StoppingCriteriaList`]. List of instances of class derived from [`StoppingCriteria`]
+ used to tell if the generation loop should stop.
logits_warper (`LogitsProcessorList`, *optional*):
- An instance of [`LogitsProcessorList`]. List of instances of class derived from
- [`LogitsWarper`] used to warp the prediction score distribution of the language
- modeling head applied before multinomial sampling at each generation step.
+ An instance of [`LogitsProcessorList`]. List of instances of class derived from [`LogitsWarper`] used
+ to warp the prediction score distribution of the language modeling head applied before multinomial
+ sampling at each generation step.
max_length (`int`, *optional*, defaults to 20):
- **DEPRECATED**. Use `logits_processor` or `stopping_criteria` directly to cap the number of
- generated tokens. The maximum length of the sequence to be generated.
+ **DEPRECATED**. Use `logits_processor` or `stopping_criteria` directly to cap the number of generated
+ tokens. The maximum length of the sequence to be generated.
pad_token_id (`int`, *optional*):
The id of the *padding* token.
eos_token_id (`int`, *optional*):
@@ -2082,16 +2074,14 @@ class GenerationMixin:
synced_gpus (`bool`, *optional*, defaults to `False`):
Whether to continue running the while loop until max_length (needed for ZeRO stage 3)
model_kwargs:
- Additional model specific kwargs will be forwarded to the `forward` function of the model. If
- model is an encoder-decoder model the kwargs should include `encoder_outputs`.
+ Additional model specific kwargs will be forwarded to the `forward` function of the model. If model is
+ an encoder-decoder model the kwargs should include `encoder_outputs`.
Return:
- [`~generation_utils.BeamSampleDecoderOnlyOutput`],
- [`~generation_utils.BeamSampleEncoderDecoderOutput`] or obj:*torch.LongTensor*: A
- `torch.LongTensor` containing the generated tokens (default behaviour) or a
- [`~generation_utils.BeamSampleDecoderOnlyOutput`] if
- `model.config.is_encoder_decoder=False` and `return_dict_in_generate=True` or a
- [`~generation_utils.BeamSampleEncoderDecoderOutput`] if
+ [`~generation_utils.BeamSampleDecoderOnlyOutput`], [`~generation_utils.BeamSampleEncoderDecoderOutput`] or
+ obj:*torch.LongTensor*: A `torch.LongTensor` containing the generated tokens (default behaviour) or a
+ [`~generation_utils.BeamSampleDecoderOnlyOutput`] if `model.config.is_encoder_decoder=False` and
+ `return_dict_in_generate=True` or a [`~generation_utils.BeamSampleEncoderDecoderOutput`] if
`model.config.is_encoder_decoder=True`.
Examples:
@@ -2351,19 +2341,17 @@ class GenerationMixin:
input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
The sequence used as a prompt for the generation.
beam_scorer (`BeamScorer`):
- An derived instance of [`BeamScorer`] that defines how beam hypotheses are
- constructed, stored and sorted during generation. For more information, the documentation of
- [`BeamScorer`] should be read.
+ An derived instance of [`BeamScorer`] that defines how beam hypotheses are constructed, stored and
+ sorted during generation. For more information, the documentation of [`BeamScorer`] should be read.
logits_processor (`LogitsProcessorList`, *optional*):
- An instance of [`LogitsProcessorList`]. List of instances of class derived from
- [`LogitsProcessor`] used to modify the prediction scores of the language modeling
- head applied at each generation step.
+ An instance of [`LogitsProcessorList`]. List of instances of class derived from [`LogitsProcessor`]
+ used to modify the prediction scores of the language modeling head applied at each generation step.
stopping_criteria (`StoppingCriteriaList`, *optional*):
- An instance of [`StoppingCriteriaList`]. List of instances of class derived from
- [`StoppingCriteria`] used to tell if the generation loop should stop.
+ An instance of [`StoppingCriteriaList`]. List of instances of class derived from [`StoppingCriteria`]
+ used to tell if the generation loop should stop.
max_length (`int`, *optional*, defaults to 20):
- **DEPRECATED**. Use `logits_processor` or `stopping_criteria` directly to cap the number of
- generated tokens. The maximum length of the sequence to be generated.
+ **DEPRECATED**. Use `logits_processor` or `stopping_criteria` directly to cap the number of generated
+ tokens. The maximum length of the sequence to be generated.
pad_token_id (`int`, *optional*):
The id of the *padding* token.
eos_token_id (`int`, *optional*):
@@ -2386,14 +2374,11 @@ class GenerationMixin:
model is an encoder-decoder model the kwargs should include `encoder_outputs`.
Return:
- [`~generation_utils.BeamSearchDecoderOnlyOutput`],
- [`~generation_utils.BeamSearchEncoderDecoderOutput`] or obj:*torch.LongTensor*: A
- `torch.LongTensor` containing the generated tokens (default behaviour) or a
- [`~generation_utils.BeamSearchDecoderOnlyOutput`] if
- [`~generation_utils.BeamSearchDecoderOnlyOutput`] if
+ [`~generation_utils.BeamSearchDecoderOnlyOutput`], [`~generation_utils.BeamSearchEncoderDecoderOutput`] or
+ obj:*torch.LongTensor*: A `torch.LongTensor` containing the generated tokens (default behaviour) or a
+ [`~generation_utils.BeamSearchDecoderOnlyOutput`] if [`~generation_utils.BeamSearchDecoderOnlyOutput`] if
`model.config.is_encoder_decoder=False` and `return_dict_in_generate=True` or a
- [`~generation_utils.BeamSearchEncoderDecoderOutput`] if
- `model.config.is_encoder_decoder=True`.
+ [`~generation_utils.BeamSearchEncoderDecoderOutput`] if `model.config.is_encoder_decoder=True`.
Examples:
diff --git a/src/transformers/image_utils.py b/src/transformers/image_utils.py
index 773f3e1cad..e94a360462 100644
--- a/src/transformers/image_utils.py
+++ b/src/transformers/image_utils.py
@@ -87,8 +87,8 @@ class ImageFeatureExtractionMixin:
def to_pil_image(self, image, rescale=None):
"""
- Converts `image` to a PIL Image. Optionally rescales it and puts the channel dimension back as the last
- axis if needed.
+ Converts `image` to a PIL Image. Optionally rescales it and puts the channel dimension back as the last axis if
+ needed.
Args:
image (`PIL.Image.Image` or `numpy.ndarray` or `torch.Tensor`):
@@ -125,8 +125,7 @@ class ImageFeatureExtractionMixin:
The image to convert to a NumPy array.
rescale (`bool`, *optional*):
Whether or not to apply the scaling factor (to make pixel values floats between 0. and 1.). Will
- default to `True` if the image is a PIL Image or an array/tensor of integers, `False`
- otherwise.
+ default to `True` if the image is a PIL Image or an array/tensor of integers, `False` otherwise.
channel_first (`bool`, *optional*, defaults to `True`):
Whether or not to permute the dimensions of the image to put the channel dimension first.
"""
@@ -151,8 +150,8 @@ class ImageFeatureExtractionMixin:
def normalize(self, image, mean, std):
"""
- Normalizes `image` with `mean` and `std`. Note that this will trigger a conversion of
- `image` to a NumPy array if it's a PIL Image.
+ Normalizes `image` with `mean` and `std`. Note that this will trigger a conversion of `image` to a NumPy array
+ if it's a PIL Image.
Args:
image (`PIL.Image.Image` or `np.ndarray` or `torch.Tensor`):
@@ -210,8 +209,8 @@ class ImageFeatureExtractionMixin:
def center_crop(self, image, size):
"""
- Crops `image` to the given size using a center crop. Note that if the image is too small to be cropped to
- the size given, it will be padded (so the returned result has the size asked).
+ Crops `image` to the given size using a center crop. Note that if the image is too small to be cropped to the
+ size given, it will be padded (so the returned result has the size asked).
Args:
image (`PIL.Image.Image` or `np.ndarray` or `torch.Tensor`):
diff --git a/src/transformers/integrations.py b/src/transformers/integrations.py
index fcaecfd658..b4fba50bad 100644
--- a/src/transformers/integrations.py
+++ b/src/transformers/integrations.py
@@ -479,15 +479,17 @@ class WandbCallback(TrainerCallback):
"""
Setup the optional Weights & Biases (*wandb*) integration.
- One can subclass and override this method to customize the setup if needed. Find more information [here](https://docs.wandb.ai/integrations/huggingface). You can also override the following environment variables:
+ One can subclass and override this method to customize the setup if needed. Find more information
+ [here](https://docs.wandb.ai/integrations/huggingface). You can also override the following environment
+ variables:
Environment:
WANDB_LOG_MODEL (`bool`, *optional*, defaults to `False`):
Whether or not to log model as artifact at the end of training. Use along with
*TrainingArguments.load_best_model_at_end* to upload best model.
WANDB_WATCH (`str`, *optional* defaults to `"gradients"`):
- Can be `"gradients"`, `"all"` or `"false"`. Set to `"false"` to disable gradient
- logging or `"all"` to log gradients and parameters.
+ Can be `"gradients"`, `"all"` or `"false"`. Set to `"false"` to disable gradient logging or `"all"` to
+ log gradients and parameters.
WANDB_PROJECT (`str`, *optional*, defaults to `"huggingface"`):
Set this to a custom string to store results in a different project.
WANDB_DISABLED (`bool`, *optional*, defaults to `False`):
@@ -608,7 +610,8 @@ class CometCallback(TrainerCallback):
Whether or not to log training assets (tf event logs, checkpoints, etc), to Comet. Can be "TRUE", or
"FALSE". Defaults to "TRUE".
- For a number of configurable items in the environment, see [here](https://www.comet.ml/docs/python-sdk/advanced/#comet-configuration-variables).
+ For a number of configurable items in the environment, see
+ [here](https://www.comet.ml/docs/python-sdk/advanced/#comet-configuration-variables).
"""
self._initialized = True
log_assets = os.getenv("COMET_LOG_ASSETS", "FALSE").upper()
@@ -705,8 +708,8 @@ class MLflowCallback(TrainerCallback):
Whether to use MLflow .log_artifact() facility to log artifacts.
This only makes sense if logging to a remote server, e.g. s3 or GCS. If set to *True* or *1*, will copy
- whatever is in [`TrainingArguments`]'s `output_dir` to the local or remote
- artifact storage. Using it without a remote storage will just copy the files to your artifact location.
+ whatever is in [`TrainingArguments`]'s `output_dir` to the local or remote artifact storage. Using it
+ without a remote storage will just copy the files to your artifact location.
"""
log_artifacts = os.getenv("HF_MLFLOW_LOG_ARTIFACTS", "FALSE").upper()
if log_artifacts in {"TRUE", "1"}:
diff --git a/src/transformers/keras_callbacks.py b/src/transformers/keras_callbacks.py
index 9960f7032b..ba9a6add8b 100644
--- a/src/transformers/keras_callbacks.py
+++ b/src/transformers/keras_callbacks.py
@@ -27,9 +27,8 @@ class KerasMetricCallback(Callback):
`eval_dataset` before being passed to the `metric_fn` in `np.ndarray` format. The `metric_fn` should compute
metrics and return a dict mapping metric names to metric values.
- We provide an example of a suitable metric_fn that computes ROUGE scores for a summarization model below.
- Note that this example skips some post-processing for readability and simplicity, and should probably
- not be used as-is!
+ We provide an example of a suitable metric_fn that computes ROUGE scores for a summarization model below. Note that
+ this example skips some post-processing for readability and simplicity, and should probably not be used as-is!
```py
from datasets import load_metric
diff --git a/src/transformers/modelcard.py b/src/transformers/modelcard.py
index 184bd80660..178f7f48c3 100644
--- a/src/transformers/modelcard.py
+++ b/src/transformers/modelcard.py
@@ -134,8 +134,8 @@ class ModelCard:
- a string, the *model id* of a pretrained model card hosted inside a model repo on huggingface.co.
Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under a
user or organization name, like `dbmdz/bert-base-german-cased`.
- - a path to a *directory* containing a model card file saved using the
- [`~ModelCard.save_pretrained`] method, e.g.: `./my_model_directory/`.
+ - a path to a *directory* containing a model card file saved using the [`~ModelCard.save_pretrained`]
+ method, e.g.: `./my_model_directory/`.
- a path or url to a saved model card JSON *file*, e.g.: `./my_model_directory/modelcard.json`.
cache_dir: (*optional*) string:
diff --git a/src/transformers/modeling_flax_outputs.py b/src/transformers/modeling_flax_outputs.py
index f2a23fa52b..b7d4bd0aec 100644
--- a/src/transformers/modeling_flax_outputs.py
+++ b/src/transformers/modeling_flax_outputs.py
@@ -28,12 +28,13 @@ class FlaxBaseModelOutput(ModelOutput):
last_hidden_state (`jnp.ndarray` of shape `(batch_size, sequence_length, hidden_size)`):
Sequence of hidden-states at the output of the last layer of the model.
hidden_states (`tuple(jnp.ndarray)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(jnp.ndarray)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -56,12 +57,13 @@ class FlaxBaseModelOutputWithPast(ModelOutput):
Dictionary of pre-computed hidden-states (key and values in the attention blocks) that can be used for fast
auto-regressive decoding. Pre-computed key and value hidden-states are of shape *[batch_size, max_length]*.
hidden_states (`tuple(jnp.ndarray)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(jnp.ndarray)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -86,12 +88,13 @@ class FlaxBaseModelOutputWithPooling(ModelOutput):
Linear layer and a Tanh activation function. The Linear layer weights are trained from the next sentence
prediction (classification) objective during pretraining.
hidden_states (`tuple(jnp.ndarray)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(jnp.ndarray)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -112,27 +115,31 @@ class FlaxBaseModelOutputWithPastAndCrossAttentions(ModelOutput):
last_hidden_state (`jnp.ndarray` of shape `(batch_size, sequence_length, hidden_size)`):
Sequence of hidden-states at the output of the last layer of the model.
- If `past_key_values` is used only the last hidden-state of the sequences of shape `(batch_size, 1, hidden_size)` is output.
+ If `past_key_values` is used only the last hidden-state of the sequences of shape `(batch_size, 1,
+ hidden_size)` is output.
past_key_values (`tuple(tuple(jnp.ndarray))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- Tuple of `tuple(jnp.ndarray)` of length `config.n_layers`, with each tuple having 2 tensors of
- shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and optionally if
- `config.is_encoder_decoder=True` 2 additional tensors of shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
+ Tuple of `tuple(jnp.ndarray)` of length `config.n_layers`, with each tuple having 2 tensors of shape
+ `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and optionally if
+ `config.is_encoder_decoder=True` 2 additional tensors of shape `(batch_size, num_heads,
+ encoder_sequence_length, embed_size_per_head)`.
Contains pre-computed hidden-states (key and values in the self-attention blocks and optionally if
- `config.is_encoder_decoder=True` in the cross-attention blocks) that can be used (see
- `past_key_values` input) to speed up sequential decoding.
+ `config.is_encoder_decoder=True` in the cross-attention blocks) that can be used (see `past_key_values`
+ input) to speed up sequential decoding.
hidden_states (`tuple(jnp.ndarray)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(jnp.ndarray)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
cross_attentions (`tuple(jnp.ndarray)`, *optional*, returned when `output_attentions=True` and `config.add_cross_attention=True` is passed or when `config.output_attentions=True`):
- Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the decoder's cross-attention layer, after the attention softmax, used to compute the
weighted average in the cross-attention heads.
@@ -155,38 +162,42 @@ class FlaxSeq2SeqModelOutput(ModelOutput):
last_hidden_state (`jnp.ndarray` of shape `(batch_size, sequence_length, hidden_size)`):
Sequence of hidden-states at the output of the last layer of the decoder of the model.
- If `past_key_values` is used only the last hidden-state of the sequences of shape `(batch_size, 1, hidden_size)` is output.
+ If `past_key_values` is used only the last hidden-state of the sequences of shape `(batch_size, 1,
+ hidden_size)` is output.
past_key_values (`tuple(tuple(jnp.ndarray))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- Tuple of `tuple(jnp.ndarray)` of length `config.n_layers`, with each tuple having 2 tensors of
- shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
- shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
+ Tuple of `tuple(jnp.ndarray)` of length `config.n_layers`, with each tuple having 2 tensors of shape
+ `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape
+ `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
decoder_hidden_states (`tuple(jnp.ndarray)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the decoder at the output of each layer plus the initial embedding outputs.
decoder_attentions (`tuple(jnp.ndarray)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the decoder, after the attention softmax, used to compute the weighted average in the
self-attention heads.
cross_attentions (`tuple(jnp.ndarray)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the decoder's cross-attention layer, after the attention softmax, used to compute the
weighted average in the cross-attention heads.
encoder_last_hidden_state (`jnp.ndarray` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
Sequence of hidden-states at the output of the last layer of the encoder of the model.
encoder_hidden_states (`tuple(jnp.ndarray)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the encoder at the output of each layer plus the initial embedding outputs.
encoder_attentions (`tuple(jnp.ndarray)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the encoder, after the attention softmax, used to compute the weighted average in the
self-attention heads.
@@ -211,24 +222,26 @@ class FlaxCausalLMOutputWithCrossAttentions(ModelOutput):
logits (`jnp.ndarray` of shape `(batch_size, sequence_length, config.vocab_size)`):
Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
hidden_states (`tuple(jnp.ndarray)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(jnp.ndarray)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
cross_attentions (`tuple(jnp.ndarray)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Cross attentions weights after the attention softmax, used to compute the weighted average in the
cross-attention heads.
past_key_values (`tuple(tuple(jnp.ndarray))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- Tuple of `jnp.ndarray` tuples of length `config.n_layers`, with each tuple containing the cached
- key, value states of the self-attention and the cross-attention layers if model is used in encoder-decoder
- setting. Only relevant if `config.is_decoder = True`.
+ Tuple of `jnp.ndarray` tuples of length `config.n_layers`, with each tuple containing the cached key, value
+ states of the self-attention and the cross-attention layers if model is used in encoder-decoder setting.
+ Only relevant if `config.is_decoder = True`.
Contains pre-computed hidden-states (key and values in the attention blocks) that can be used (see
`past_key_values` input) to speed up sequential decoding.
@@ -250,12 +263,13 @@ class FlaxMaskedLMOutput(ModelOutput):
logits (`jnp.ndarray` of shape `(batch_size, sequence_length, config.vocab_size)`):
Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
hidden_states (`tuple(jnp.ndarray)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(jnp.ndarray)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -278,36 +292,39 @@ class FlaxSeq2SeqLMOutput(ModelOutput):
logits (`jnp.ndarray` of shape `(batch_size, sequence_length, config.vocab_size)`):
Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
past_key_values (`tuple(tuple(jnp.ndarray))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- Tuple of `tuple(jnp.ndarray)` of length `config.n_layers`, with each tuple having 2 tensors of
- shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
- shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
+ Tuple of `tuple(jnp.ndarray)` of length `config.n_layers`, with each tuple having 2 tensors of shape
+ `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape
+ `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
decoder_hidden_states (`tuple(jnp.ndarray)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the decoder at the output of each layer plus the initial embedding outputs.
decoder_attentions (`tuple(jnp.ndarray)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the decoder, after the attention softmax, used to compute the weighted average in the
self-attention heads.
cross_attentions (`tuple(jnp.ndarray)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the decoder's cross-attention layer, after the attention softmax, used to compute the
weighted average in the cross-attention heads.
encoder_last_hidden_state (`jnp.ndarray` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
Sequence of hidden-states at the output of the last layer of the encoder of the model.
encoder_hidden_states (`tuple(jnp.ndarray)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the encoder at the output of each layer plus the initial embedding outputs.
encoder_attentions (`tuple(jnp.ndarray)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the encoder, after the attention softmax, used to compute the weighted average in the
self-attention heads.
@@ -333,12 +350,13 @@ class FlaxNextSentencePredictorOutput(ModelOutput):
Prediction scores of the next sequence prediction (classification) head (scores of True/False continuation
before SoftMax).
hidden_states (`tuple(jnp.ndarray)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(jnp.ndarray)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -358,12 +376,13 @@ class FlaxSequenceClassifierOutput(ModelOutput):
logits (`jnp.ndarray` of shape `(batch_size, config.num_labels)`):
Classification (or regression if config.num_labels==1) scores (before SoftMax).
hidden_states (`tuple(jnp.ndarray)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(jnp.ndarray)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -383,36 +402,39 @@ class FlaxSeq2SeqSequenceClassifierOutput(ModelOutput):
logits (`jnp.ndarray` of shape `(batch_size, config.num_labels)`):
Classification (or regression if config.num_labels==1) scores (before SoftMax).
past_key_values (`tuple(tuple(jnp.ndarray))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- Tuple of `tuple(jnp.ndarray)` of length `config.n_layers`, with each tuple having 2 tensors of
- shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
- shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
+ Tuple of `tuple(jnp.ndarray)` of length `config.n_layers`, with each tuple having 2 tensors of shape
+ `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape
+ `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
decoder_hidden_states (`tuple(jnp.ndarray)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the decoder at the output of each layer plus the initial embedding outputs.
decoder_attentions (`tuple(jnp.ndarray)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the decoder, after the attention softmax, used to compute the weighted average in the
self-attention heads.
cross_attentions (`tuple(jnp.ndarray)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the decoder's cross-attention layer, after the attention softmax, used to compute the
weighted average in the cross-attention heads.
encoder_last_hidden_state (`jnp.ndarray` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
Sequence of hidden-states at the output of the last layer of the encoder of the model.
encoder_hidden_states (`tuple(jnp.ndarray)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the encoder at the output of each layer plus the initial embedding outputs.
encoder_attentions (`tuple(jnp.ndarray)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the encoder, after the attention softmax, used to compute the weighted average in the
self-attention heads.
@@ -439,12 +461,13 @@ class FlaxMultipleChoiceModelOutput(ModelOutput):
Classification scores (before SoftMax).
hidden_states (`tuple(jnp.ndarray)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(jnp.ndarray)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -464,12 +487,13 @@ class FlaxTokenClassifierOutput(ModelOutput):
logits (`jnp.ndarray` of shape `(batch_size, sequence_length, config.num_labels)`):
Classification scores (before SoftMax).
hidden_states (`tuple(jnp.ndarray)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(jnp.ndarray)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -491,12 +515,13 @@ class FlaxQuestionAnsweringModelOutput(ModelOutput):
end_logits (`jnp.ndarray` of shape `(batch_size, sequence_length)`):
Span-end scores (before SoftMax).
hidden_states (`tuple(jnp.ndarray)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(jnp.ndarray)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -519,36 +544,39 @@ class FlaxSeq2SeqQuestionAnsweringModelOutput(ModelOutput):
end_logits (`jnp.ndarray` of shape `(batch_size, sequence_length)`):
Span-end scores (before SoftMax).
past_key_values (`tuple(tuple(jnp.ndarray))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- Tuple of `tuple(jnp.ndarray)` of length `config.n_layers`, with each tuple having 2 tensors of
- shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
- shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
+ Tuple of `tuple(jnp.ndarray)` of length `config.n_layers`, with each tuple having 2 tensors of shape
+ `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape
+ `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
decoder_hidden_states (`tuple(jnp.ndarray)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the decoder at the output of each layer plus the initial embedding outputs.
decoder_attentions (`tuple(jnp.ndarray)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the decoder, after the attention softmax, used to compute the weighted average in the
self-attention heads.
cross_attentions (`tuple(jnp.ndarray)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the decoder's cross-attention layer, after the attention softmax, used to compute the
weighted average in the cross-attention heads.
encoder_last_hidden_state (`jnp.ndarray` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
Sequence of hidden-states at the output of the last layer of the encoder of the model.
encoder_hidden_states (`tuple(jnp.ndarray)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the encoder at the output of each layer plus the initial embedding outputs.
encoder_attentions (`tuple(jnp.ndarray)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the encoder, after the attention softmax, used to compute the weighted average in the
self-attention heads.
diff --git a/src/transformers/modeling_flax_utils.py b/src/transformers/modeling_flax_utils.py
index 945349f361..0c1f17437f 100644
--- a/src/transformers/modeling_flax_utils.py
+++ b/src/transformers/modeling_flax_utils.py
@@ -67,17 +67,17 @@ class FlaxPreTrainedModel(PushToHubMixin, FlaxGenerationMixin):
r"""
Base class for all models.
- [`FlaxPreTrainedModel`] takes care of storing the configuration of the models and handles
- methods for loading, downloading and saving models.
+ [`FlaxPreTrainedModel`] takes care of storing the configuration of the models and handles methods for loading,
+ downloading and saving models.
Class attributes (overridden by derived classes):
- - **config_class** ([`PretrainedConfig`]) -- A subclass of
- [`PretrainedConfig`] to use as configuration class for this model architecture.
- - **base_model_prefix** (`str`) -- A string indicating the attribute associated to the base model in
- derived classes of the same architecture adding modules on top of the base model.
- - **main_input_name** (`str`) -- The name of the principal input to the model (often `input_ids` for
- NLP models, `pixel_values` for vision models and `input_values` for speech models).
+ - **config_class** ([`PretrainedConfig`]) -- A subclass of [`PretrainedConfig`] to use as configuration class
+ for this model architecture.
+ - **base_model_prefix** (`str`) -- A string indicating the attribute associated to the base model in derived
+ classes of the same architecture adding modules on top of the base model.
+ - **main_input_name** (`str`) -- The name of the principal input to the model (often `input_ids` for NLP
+ models, `pixel_values` for vision models and `input_values` for speech models).
"""
config_class = None
base_model_prefix = ""
@@ -183,8 +183,8 @@ class FlaxPreTrainedModel(PushToHubMixin, FlaxGenerationMixin):
def to_bf16(self, params: Union[Dict, FrozenDict], mask: Any = None):
r"""
- Cast the floating-point `params` to `jax.numpy.bfloat16`. This returns a new `params` tree and does not
- cast the `params` in place.
+ Cast the floating-point `params` to `jax.numpy.bfloat16`. This returns a new `params` tree and does not cast
+ the `params` in place.
This method can be used on TPU to explicitly convert the model parameters to bfloat16 precision to do full
half-precision training or to save weights in bfloat16 for inference in order to save memory and improve speed.
@@ -193,8 +193,8 @@ class FlaxPreTrainedModel(PushToHubMixin, FlaxGenerationMixin):
params (`Union[Dict, FrozenDict]`):
A `PyTree` of model parameters.
mask (`Union[Dict, FrozenDict]`):
- A `PyTree` with same structure as the `params` tree. The leaves should be booleans, `True` for
- params you want to cast, and should be `False` for those you want to skip.
+ A `PyTree` with same structure as the `params` tree. The leaves should be booleans, `True` for params
+ you want to cast, and should be `False` for those you want to skip.
Examples:
@@ -218,15 +218,14 @@ class FlaxPreTrainedModel(PushToHubMixin, FlaxGenerationMixin):
def to_fp32(self, params: Union[Dict, FrozenDict], mask: Any = None):
r"""
Cast the floating-point `parmas` to `jax.numpy.float32`. This method can be used to explicitly convert the
- model parameters to fp32 precision. This returns a new `params` tree and does not cast the `params` in
- place.
+ model parameters to fp32 precision. This returns a new `params` tree and does not cast the `params` in place.
Arguments:
params (`Union[Dict, FrozenDict]`):
A `PyTree` of model parameters.
mask (`Union[Dict, FrozenDict]`):
- A `PyTree` with same structure as the `params` tree. The leaves should be booleans, `True` for
- params you want to cast, and should be `False` for those you want to skip
+ A `PyTree` with same structure as the `params` tree. The leaves should be booleans, `True` for params
+ you want to cast, and should be `False` for those you want to skip
Examples:
@@ -244,8 +243,8 @@ class FlaxPreTrainedModel(PushToHubMixin, FlaxGenerationMixin):
def to_fp16(self, params: Union[Dict, FrozenDict], mask: Any = None):
r"""
- Cast the floating-point `parmas` to `jax.numpy.float16`. This returns a new `params` tree and does not
- cast the `params` in place.
+ Cast the floating-point `parmas` to `jax.numpy.float16`. This returns a new `params` tree and does not cast the
+ `params` in place.
This method can be used on GPU to explicitly convert the model parameters to float16 precision to do full
half-precision training or to save weights in float16 for inference in order to save memory and improve speed.
@@ -254,8 +253,8 @@ class FlaxPreTrainedModel(PushToHubMixin, FlaxGenerationMixin):
params (`Union[Dict, FrozenDict]`):
A `PyTree` of model parameters.
mask (`Union[Dict, FrozenDict]`):
- A `PyTree` with same structure as the `params` tree. The leaves should be booleans, `True` for
- params you want to cast, and should be `False` for those you want to skip
+ A `PyTree` with same structure as the `params` tree. The leaves should be booleans, `True` for params
+ you want to cast, and should be `False` for those you want to skip
Examples:
@@ -300,15 +299,15 @@ class FlaxPreTrainedModel(PushToHubMixin, FlaxGenerationMixin):
Can be either:
- A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co.
- Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under
- a user or organization name, like `dbmdz/bert-base-german-cased`.
+ Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under a
+ user or organization name, like `dbmdz/bert-base-german-cased`.
- A path to a *directory* containing model weights saved using
[`~FlaxPreTrainedModel.save_pretrained`], e.g., `./my_model_directory/`.
- - A path or url to a *pt index checkpoint file* (e.g, `./tf_model/model.ckpt.index`). In this
- case, `from_pt` should be set to `True`.
+ - A path or url to a *pt index checkpoint file* (e.g, `./tf_model/model.ckpt.index`). In this case,
+ `from_pt` should be set to `True`.
dtype (`jax.numpy.dtype`, *optional*, defaults to `jax.numpy.float32`):
- The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on
- GPUs) and `jax.numpy.bfloat16` (on TPUs).
+ The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on GPUs) and
+ `jax.numpy.bfloat16` (on TPUs).
This can be used to enable mixed-precision training or half-precision inference on GPUs or TPUs. If
specified all the computation will be performed with the given `dtype`.
@@ -316,8 +315,7 @@ class FlaxPreTrainedModel(PushToHubMixin, FlaxGenerationMixin):
**Note that this only specifies the dtype of the computation and does not influence the dtype of model
parameters.**
- If you wish to change the dtype of the model parameters, see
- [`~FlaxPreTrainedModel.to_fp16`] and
+ If you wish to change the dtype of the model parameters, see [`~FlaxPreTrainedModel.to_fp16`] and
[`~FlaxPreTrainedModel.to_bf16`].
model_args (sequence of positional arguments, *optional*):
All remaining positional arguments will be passed to the underlying model's `__init__` method.
@@ -332,8 +330,8 @@ class FlaxPreTrainedModel(PushToHubMixin, FlaxGenerationMixin):
- The model is a model provided by the library (loaded with the *model id* string of a pretrained
model).
- - The model was saved using [`~PreTrainedModel.save_pretrained`] and is reloaded
- by supplying the save directory.
+ - The model was saved using [`~PreTrainedModel.save_pretrained`] and is reloaded by supplying the
+ save directory.
- The model is loaded by supplying a local directory as `pretrained_model_name_or_path` and a
configuration JSON file named *config.json* is found in the directory.
cache_dir (`Union[str, os.PathLike]`, *optional*):
@@ -353,7 +351,8 @@ class FlaxPreTrainedModel(PushToHubMixin, FlaxGenerationMixin):
Whether or not to delete incompletely received files. Will attempt to resume the download if such a
file exists.
proxies (`Dict[str, str]`, *optional*):
- A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request.
+ A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',
+ 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request.
local_files_only(`bool`, *optional*, defaults to `False`):
Whether or not to only look at local files (i.e., do not try to download the model).
revision(`str`, *optional*, defaults to `"main"`):
@@ -369,10 +368,10 @@ class FlaxPreTrainedModel(PushToHubMixin, FlaxGenerationMixin):
underlying model's `__init__` method (we assume all relevant updates to the configuration have
already been done)
- If a configuration is not provided, `kwargs` will be first passed to the configuration class
- initialization function ([`~PretrainedConfig.from_pretrained`]). Each key of
- `kwargs` that corresponds to a configuration attribute will be used to override said attribute
- with the supplied `kwargs` value. Remaining keys that do not correspond to any configuration
- attribute will be passed to the underlying model's `__init__` function.
+ initialization function ([`~PretrainedConfig.from_pretrained`]). Each key of `kwargs` that
+ corresponds to a configuration attribute will be used to override said attribute with the
+ supplied `kwargs` value. Remaining keys that do not correspond to any configuration attribute
+ will be passed to the underlying model's `__init__` function.
Examples:
@@ -605,16 +604,14 @@ class FlaxPreTrainedModel(PushToHubMixin, FlaxGenerationMixin):
- Using `push_to_hub=True` will synchronize the repository you are pushing to with
- `save_directory`, which requires `save_directory` to be a local clone of the repo you are
- pushing to if it's an existing folder. Pass along `temp_dir=True` to use a temporary directory
- instead.
+ Using `push_to_hub=True` will synchronize the repository you are pushing to with `save_directory`,
+ which requires `save_directory` to be a local clone of the repo you are pushing to if it's an existing
+ folder. Pass along `temp_dir=True` to use a temporary directory instead.
kwargs:
- Additional key word arguments passed along to the
- [`~file_utils.PushToHubMixin.push_to_hub`] method.
+ Additional key word arguments passed along to the [`~file_utils.PushToHubMixin.push_to_hub`] method.
"""
if os.path.isfile(save_directory):
logger.error(f"Provided path ({save_directory}) should be a directory, not a file")
diff --git a/src/transformers/modeling_outputs.py b/src/transformers/modeling_outputs.py
index 34bed2119b..c7f4a27fb3 100644
--- a/src/transformers/modeling_outputs.py
+++ b/src/transformers/modeling_outputs.py
@@ -29,12 +29,13 @@ class BaseModelOutput(ModelOutput):
last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
Sequence of hidden-states at the output of the last layer of the model.
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -59,12 +60,13 @@ class BaseModelOutputWithPooling(ModelOutput):
the classification token after processing through a linear layer and a tanh activation function. The linear
layer weights are trained from the next sentence prediction (classification) objective during pretraining.
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -85,22 +87,25 @@ class BaseModelOutputWithPast(ModelOutput):
last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
Sequence of hidden-states at the output of the last layer of the model.
- If `past_key_values` is used only the last hidden-state of the sequences of shape `(batch_size, 1, hidden_size)` is output.
+ If `past_key_values` is used only the last hidden-state of the sequences of shape `(batch_size, 1,
+ hidden_size)` is output.
past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors
- of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and optionally if
- `config.is_encoder_decoder=True` 2 additional tensors of shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
+ Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
+ `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and optionally if
+ `config.is_encoder_decoder=True` 2 additional tensors of shape `(batch_size, num_heads,
+ encoder_sequence_length, embed_size_per_head)`.
Contains pre-computed hidden-states (key and values in the self-attention blocks and optionally if
- `config.is_encoder_decoder=True` in the cross-attention blocks) that can be used (see
- `past_key_values` input) to speed up sequential decoding.
+ `config.is_encoder_decoder=True` in the cross-attention blocks) that can be used (see `past_key_values`
+ input) to speed up sequential decoding.
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -121,17 +126,19 @@ class BaseModelOutputWithCrossAttentions(ModelOutput):
last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
Sequence of hidden-states at the output of the last layer of the model.
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
cross_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` and `config.add_cross_attention=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the decoder's cross-attention layer, after the attention softmax, used to compute the
weighted average in the cross-attention heads.
@@ -157,28 +164,31 @@ class BaseModelOutputWithPoolingAndCrossAttentions(ModelOutput):
the classification token after processing through a linear layer and a tanh activation function. The linear
layer weights are trained from the next sentence prediction (classification) objective during pretraining.
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
cross_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` and `config.add_cross_attention=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the decoder's cross-attention layer, after the attention softmax, used to compute the
weighted average in the cross-attention heads.
past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors
- of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and optionally if
- `config.is_encoder_decoder=True` 2 additional tensors of shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
+ Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
+ `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and optionally if
+ `config.is_encoder_decoder=True` 2 additional tensors of shape `(batch_size, num_heads,
+ encoder_sequence_length, embed_size_per_head)`.
Contains pre-computed hidden-states (key and values in the self-attention blocks and optionally if
- `config.is_encoder_decoder=True` in the cross-attention blocks) that can be used (see
- `past_key_values` input) to speed up sequential decoding.
+ `config.is_encoder_decoder=True` in the cross-attention blocks) that can be used (see `past_key_values`
+ input) to speed up sequential decoding.
"""
last_hidden_state: torch.FloatTensor = None
@@ -198,27 +208,31 @@ class BaseModelOutputWithPastAndCrossAttentions(ModelOutput):
last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
Sequence of hidden-states at the output of the last layer of the model.
- If `past_key_values` is used only the last hidden-state of the sequences of shape `(batch_size, 1, hidden_size)` is output.
+ If `past_key_values` is used only the last hidden-state of the sequences of shape `(batch_size, 1,
+ hidden_size)` is output.
past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors
- of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and optionally if
- `config.is_encoder_decoder=True` 2 additional tensors of shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
+ Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
+ `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and optionally if
+ `config.is_encoder_decoder=True` 2 additional tensors of shape `(batch_size, num_heads,
+ encoder_sequence_length, embed_size_per_head)`.
Contains pre-computed hidden-states (key and values in the self-attention blocks and optionally if
- `config.is_encoder_decoder=True` in the cross-attention blocks) that can be used (see
- `past_key_values` input) to speed up sequential decoding.
+ `config.is_encoder_decoder=True` in the cross-attention blocks) that can be used (see `past_key_values`
+ input) to speed up sequential decoding.
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
cross_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` and `config.add_cross_attention=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the decoder's cross-attention layer, after the attention softmax, used to compute the
weighted average in the cross-attention heads.
@@ -241,38 +255,42 @@ class Seq2SeqModelOutput(ModelOutput):
last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
Sequence of hidden-states at the output of the last layer of the decoder of the model.
- If `past_key_values` is used only the last hidden-state of the sequences of shape `(batch_size, 1, hidden_size)` is output.
+ If `past_key_values` is used only the last hidden-state of the sequences of shape `(batch_size, 1,
+ hidden_size)` is output.
past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors
- of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
- shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
+ Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
+ `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape
+ `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
decoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the decoder at the output of each layer plus the initial embedding outputs.
decoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the decoder, after the attention softmax, used to compute the weighted average in the
self-attention heads.
cross_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the decoder's cross-attention layer, after the attention softmax, used to compute the
weighted average in the cross-attention heads.
encoder_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
Sequence of hidden-states at the output of the last layer of the encoder of the model.
encoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the encoder at the output of each layer plus the initial embedding outputs.
encoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the encoder, after the attention softmax, used to compute the weighted average in the
self-attention heads.
@@ -299,12 +317,13 @@ class CausalLMOutput(ModelOutput):
logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -327,18 +346,19 @@ class CausalLMOutputWithPast(ModelOutput):
logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors
- of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`)
+ Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
+ `(batch_size, num_heads, sequence_length, embed_size_per_head)`)
Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
`past_key_values` input) to speed up sequential decoding.
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -362,24 +382,26 @@ class CausalLMOutputWithCrossAttentions(ModelOutput):
logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
cross_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Cross attentions weights after the attention softmax, used to compute the weighted average in the
cross-attention heads.
past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- Tuple of `torch.FloatTensor` tuples of length `config.n_layers`, with each tuple containing the
- cached key, value states of the self-attention and the cross-attention layers if model is used in
- encoder-decoder setting. Only relevant if `config.is_decoder = True`.
+ Tuple of `torch.FloatTensor` tuples of length `config.n_layers`, with each tuple containing the cached key,
+ value states of the self-attention and the cross-attention layers if model is used in encoder-decoder
+ setting. Only relevant if `config.is_decoder = True`.
Contains pre-computed hidden-states (key and values in the attention blocks) that can be used (see
`past_key_values` input) to speed up sequential decoding.
@@ -404,18 +426,19 @@ class SequenceClassifierOutputWithPast(ModelOutput):
logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
Classification (or regression if config.num_labels==1) scores (before SoftMax).
past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors
- of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`)
+ Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
+ `(batch_size, num_heads, sequence_length, embed_size_per_head)`)
Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
`past_key_values` input) to speed up sequential decoding.
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -439,12 +462,13 @@ class MaskedLMOutput(ModelOutput):
logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -467,36 +491,39 @@ class Seq2SeqLMOutput(ModelOutput):
logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors
- of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
- shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
+ Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
+ `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape
+ `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
decoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the decoder at the output of each layer plus the initial embedding outputs.
decoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the decoder, after the attention softmax, used to compute the weighted average in the
self-attention heads.
cross_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the decoder's cross-attention layer, after the attention softmax, used to compute the
weighted average in the cross-attention heads.
encoder_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
Sequence of hidden-states at the output of the last layer of the encoder of the model.
encoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the encoder at the output of each layer plus the initial embedding outputs.
encoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the encoder, after the attention softmax, used to compute the weighted average in the
self-attention heads.
@@ -525,12 +552,13 @@ class NextSentencePredictorOutput(ModelOutput):
Prediction scores of the next sequence prediction (classification) head (scores of True/False continuation
before SoftMax).
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -553,12 +581,13 @@ class SequenceClassifierOutput(ModelOutput):
logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
Classification (or regression if config.num_labels==1) scores (before SoftMax).
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -581,36 +610,39 @@ class Seq2SeqSequenceClassifierOutput(ModelOutput):
logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
Classification (or regression if config.num_labels==1) scores (before SoftMax).
past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors
- of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
- shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
+ Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
+ `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape
+ `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
decoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the decoder at the output of each layer plus the initial embedding outputs.
decoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the decoder, after the attention softmax, used to compute the weighted average in the
self-attention heads.
cross_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the decoder's cross-attention layer, after the attention softmax, used to compute the
weighted average in the cross-attention heads.
encoder_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
Sequence of hidden-states at the output of the last layer of the encoder of the model.
encoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the encoder at the output of each layer plus the initial embedding outputs.
encoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the encoder, after the attention softmax, used to compute the weighted average in the
self-attention heads.
@@ -640,12 +672,13 @@ class MultipleChoiceModelOutput(ModelOutput):
Classification scores (before SoftMax).
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -668,12 +701,13 @@ class TokenClassifierOutput(ModelOutput):
logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.num_labels)`):
Classification scores (before SoftMax).
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -698,12 +732,13 @@ class QuestionAnsweringModelOutput(ModelOutput):
end_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length)`):
Span-end scores (before SoftMax).
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -729,36 +764,39 @@ class Seq2SeqQuestionAnsweringModelOutput(ModelOutput):
end_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length)`):
Span-end scores (before SoftMax).
past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors
- of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
- shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
+ Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
+ `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape
+ `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
decoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the decoder at the output of each layer plus the initial embedding outputs.
decoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the decoder, after the attention softmax, used to compute the weighted average in the
self-attention heads.
cross_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the decoder's cross-attention layer, after the attention softmax, used to compute the
weighted average in the cross-attention heads.
encoder_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
Sequence of hidden-states at the output of the last layer of the encoder of the model.
encoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the encoder at the output of each layer plus the initial embedding outputs.
encoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the encoder, after the attention softmax, used to compute the weighted average in the
self-attention heads.
diff --git a/src/transformers/modeling_tf_outputs.py b/src/transformers/modeling_tf_outputs.py
index 581f491243..5c089d0165 100644
--- a/src/transformers/modeling_tf_outputs.py
+++ b/src/transformers/modeling_tf_outputs.py
@@ -29,12 +29,13 @@ class TFBaseModelOutput(ModelOutput):
last_hidden_state (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`):
Sequence of hidden-states at the output of the last layer of the model.
hidden_states (`tuple(tf.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -61,12 +62,13 @@ class TFBaseModelOutputWithPooling(ModelOutput):
This output is usually *not* a good summary of the semantic content of the input, you're often better with
averaging or pooling the sequence of hidden-states for the whole input sequence.
hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -94,22 +96,25 @@ class TFBaseModelOutputWithPoolingAndCrossAttentions(ModelOutput):
This output is usually *not* a good summary of the semantic content of the input, you're often better with
averaging or pooling the sequence of hidden-states for the whole input sequence.
past_key_values (`List[tf.Tensor]`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- List of `tf.Tensor` of length `config.n_layers`, with each tensor of shape `(2, batch_size, num_heads, sequence_length, embed_size_per_head)`).
+ List of `tf.Tensor` of length `config.n_layers`, with each tensor of shape `(2, batch_size, num_heads,
+ sequence_length, embed_size_per_head)`).
Contains pre-computed hidden-states (key and values in the attention blocks) that can be used (see
`past_key_values` input) to speed up sequential decoding.
hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
cross_attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the decoder's cross-attention layer, after the attention softmax, used to compute the
weighted average in the cross-attention heads.
@@ -132,19 +137,22 @@ class TFBaseModelOutputWithPast(ModelOutput):
last_hidden_state (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`):
Sequence of hidden-states at the output of the last layer of the model.
- If `past_key_values` is used only the last hidden-state of the sequences of shape `(batch_size, 1, hidden_size)` is output.
+ If `past_key_values` is used only the last hidden-state of the sequences of shape `(batch_size, 1,
+ hidden_size)` is output.
past_key_values (`List[tf.Tensor]`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- List of `tf.Tensor` of length `config.n_layers`, with each tensor of shape `(2, batch_size, num_heads, sequence_length, embed_size_per_head)`).
+ List of `tf.Tensor` of length `config.n_layers`, with each tensor of shape `(2, batch_size, num_heads,
+ sequence_length, embed_size_per_head)`).
Contains pre-computed hidden-states (key and values in the attention blocks) that can be used (see
`past_key_values` input) to speed up sequential decoding.
hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -165,17 +173,19 @@ class TFBaseModelOutputWithCrossAttentions(ModelOutput):
last_hidden_state (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`):
Sequence of hidden-states at the output of the last layer of the model.
hidden_states (`tuple(tf.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
cross_attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the decoder's cross-attention layer, after the attention softmax, used to compute the
weighted average in the cross-attention heads.
@@ -196,24 +206,28 @@ class TFBaseModelOutputWithPastAndCrossAttentions(ModelOutput):
last_hidden_state (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`):
Sequence of hidden-states at the output of the last layer of the model.
- If `past_key_values` is used only the last hidden-state of the sequences of shape `(batch_size, 1, hidden_size)` is output.
+ If `past_key_values` is used only the last hidden-state of the sequences of shape `(batch_size, 1,
+ hidden_size)` is output.
past_key_values (`List[tf.Tensor]`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- List of `tf.Tensor` of length `config.n_layers`, with each tensor of shape `(2, batch_size, num_heads, sequence_length, embed_size_per_head)`).
+ List of `tf.Tensor` of length `config.n_layers`, with each tensor of shape `(2, batch_size, num_heads,
+ sequence_length, embed_size_per_head)`).
Contains pre-computed hidden-states (key and values in the attention blocks) that can be used (see
`past_key_values` input) to speed up sequential decoding.
hidden_states (`tuple(tf.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
cross_attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the decoder's cross-attention layer, after the attention softmax, used to compute the
weighted average in the cross-attention heads.
@@ -236,36 +250,41 @@ class TFSeq2SeqModelOutput(ModelOutput):
last_hidden_state (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`):
Sequence of hidden-states at the output of the last layer of the decoder of the model.
- If `past_key_values` is used only the last hidden-state of the sequences of shape `(batch_size, 1, hidden_size)` is output.
+ If `past_key_values` is used only the last hidden-state of the sequences of shape `(batch_size, 1,
+ hidden_size)` is output.
past_key_values (`List[tf.Tensor]`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- List of `tf.Tensor` of length `config.n_layers`, with each tensor of shape `(2, batch_size, num_heads, sequence_length, embed_size_per_head)`).
+ List of `tf.Tensor` of length `config.n_layers`, with each tensor of shape `(2, batch_size, num_heads,
+ sequence_length, embed_size_per_head)`).
Contains pre-computed hidden-states (key and values in the attention blocks) of the decoder that can be
used (see `past_key_values` input) to speed up sequential decoding.
decoder_hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the decoder at the output of each layer plus the initial embedding outputs.
decoder_attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the decoder, after the attention softmax, used to compute the weighted average in the
self-attention heads.
cross_attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the decoder's cross-attention layer, after the attention softmax, used to compute the
weighted average in the cross-attention heads.
encoder_last_hidden_state (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
Sequence of hidden-states at the output of the last layer of the encoder of the model.
encoder_hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the encoder at the output of each layer plus the initial embedding outputs.
encoder_attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the encoder, after the attention softmax, used to compute the weighted average in the
self-attention heads.
@@ -292,12 +311,13 @@ class TFCausalLMOutput(ModelOutput):
logits (`tf.Tensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -320,17 +340,19 @@ class TFCausalLMOutputWithPast(ModelOutput):
logits (`tf.Tensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
past_key_values (`List[tf.Tensor]`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- List of `tf.Tensor` of length `config.n_layers`, with each tensor of shape `(2, batch_size, num_heads, sequence_length, embed_size_per_head)`).
+ List of `tf.Tensor` of length `config.n_layers`, with each tensor of shape `(2, batch_size, num_heads,
+ sequence_length, embed_size_per_head)`).
Contains pre-computed hidden-states (key and values in the attention blocks) that can be used (see
`past_key_values` input) to speed up sequential decoding.
hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -354,22 +376,25 @@ class TFCausalLMOutputWithCrossAttentions(ModelOutput):
logits (`tf.Tensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
cross_attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the decoder's cross-attention layer, after the attention softmax, used to compute the
weighted average in the cross-attention heads.
past_key_values (`List[tf.Tensor]`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- List of `tf.Tensor` of length `config.n_layers`, with each tensor of shape `(2, batch_size, num_heads, sequence_length, embed_size_per_head)`).
+ List of `tf.Tensor` of length `config.n_layers`, with each tensor of shape `(2, batch_size, num_heads,
+ sequence_length, embed_size_per_head)`).
Contains pre-computed hidden-states (key and values in the attention blocks) that can be used (see
`past_key_values` input) to speed up sequential decoding.
@@ -394,12 +419,13 @@ class TFMaskedLMOutput(ModelOutput):
logits (`tf.Tensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -422,34 +448,38 @@ class TFSeq2SeqLMOutput(ModelOutput):
logits (`tf.Tensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
past_key_values (`List[tf.Tensor]`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- List of `tf.Tensor` of length `config.n_layers`, with each tensor of shape `(2, batch_size, num_heads, sequence_length, embed_size_per_head)`).
+ List of `tf.Tensor` of length `config.n_layers`, with each tensor of shape `(2, batch_size, num_heads,
+ sequence_length, embed_size_per_head)`).
Contains pre-computed hidden-states (key and values in the attention blocks) of the decoder that can be
used (see `past_key_values` input) to speed up sequential decoding.
decoder_hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the decoder at the output of each layer plus the initial embedding outputs.
decoder_attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the decoder, after the attention softmax, used to compute the weighted average in the
self-attention heads.
cross_attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the decoder's cross-attention layer, after the attention softmax, used to compute the
weighted average in the cross-attention heads.
encoder_last_hidden_state (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
Sequence of hidden-states at the output of the last layer of the encoder of the model.
encoder_hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the encoder at the output of each layer plus the initial embedding outputs.
encoder_attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the encoder, after the attention softmax, used to compute the weighted average in the
self-attention heads.
@@ -478,12 +508,13 @@ class TFNextSentencePredictorOutput(ModelOutput):
Prediction scores of the next sequence prediction (classification) head (scores of True/False continuation
before SoftMax).
hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -506,12 +537,13 @@ class TFSequenceClassifierOutput(ModelOutput):
logits (`tf.Tensor` of shape `(batch_size, config.num_labels)`):
Classification (or regression if config.num_labels==1) scores (before SoftMax).
hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -534,29 +566,32 @@ class TFSeq2SeqSequenceClassifierOutput(ModelOutput):
logits (`tf.Tensor` of shape `(batch_size, config.num_labels)`):
Classification (or regression if config.num_labels==1) scores (before SoftMax).
past_key_values (`List[tf.Tensor]`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- List of `tf.Tensor` of length `config.n_layers`, with each tensor of shape `(2, batch_size, num_heads, sequence_length, embed_size_per_head)`).
+ List of `tf.Tensor` of length `config.n_layers`, with each tensor of shape `(2, batch_size, num_heads,
+ sequence_length, embed_size_per_head)`).
Contains pre-computed hidden-states (key and values in the attention blocks) of the decoder that can be
used (see `past_key_values` input) to speed up sequential decoding.
decoder_hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the decoder at the output of each layer plus the initial embedding outputs.
decoder_attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the decoder, after the attention softmax, used to compute the weighted average in the
self-attention heads.
encoder_last_hidden_state (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
Sequence of hidden-states at the output of the last layer of the encoder of the model.
encoder_hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the encoder at the output of each layer plus the initial embedding outputs.
encoder_attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the encoder, after the attention softmax, used to compute the weighted average in the
self-attention heads.
@@ -585,12 +620,13 @@ class TFMultipleChoiceModelOutput(ModelOutput):
Classification scores (before SoftMax).
hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -613,12 +649,13 @@ class TFTokenClassifierOutput(ModelOutput):
logits (`tf.Tensor` of shape `(batch_size, sequence_length, config.num_labels)`):
Classification scores (before SoftMax).
hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -643,12 +680,13 @@ class TFQuestionAnsweringModelOutput(ModelOutput):
end_logits (`tf.Tensor` of shape `(batch_size, sequence_length)`):
Span-end scores (before SoftMax).
hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -674,29 +712,32 @@ class TFSeq2SeqQuestionAnsweringModelOutput(ModelOutput):
end_logits (`tf.Tensor` of shape `(batch_size, sequence_length)`):
Span-end scores (before SoftMax).
past_key_values (`List[tf.Tensor]`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- List of `tf.Tensor` of length `config.n_layers`, with each tensor of shape `(2, batch_size, num_heads, sequence_length, embed_size_per_head)`).
+ List of `tf.Tensor` of length `config.n_layers`, with each tensor of shape `(2, batch_size, num_heads,
+ sequence_length, embed_size_per_head)`).
Contains pre-computed hidden-states (key and values in the attention blocks) of the decoder that can be
used (see `past_key_values` input) to speed up sequential decoding.
decoder_hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the decoder at the output of each layer plus the initial embedding outputs.
decoder_attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the decoder, after the attention softmax, used to compute the weighted average in the
self-attention heads.
encoder_last_hidden_state (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
Sequence of hidden-states at the output of the last layer of the encoder of the model.
encoder_hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the encoder at the output of each layer plus the initial embedding outputs.
encoder_attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the encoder, after the attention softmax, used to compute the weighted average in the
self-attention heads.
@@ -724,17 +765,19 @@ class TFSequenceClassifierOutputWithPast(ModelOutput):
logits (`tf.Tensor` of shape `(batch_size, config.num_labels)`):
Classification (or regression if config.num_labels==1) scores (before SoftMax).
past_key_values (`List[tf.Tensor]`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- List of `tf.Tensor` of length `config.n_layers`, with each tensor of shape `(2, batch_size, num_heads, sequence_length, embed_size_per_head)`).
+ List of `tf.Tensor` of length `config.n_layers`, with each tensor of shape `(2, batch_size, num_heads,
+ sequence_length, embed_size_per_head)`).
Contains pre-computed hidden-states (key and values in the attention blocks) that can be used (see
`past_key_values` input) to speed up sequential decoding.
hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
diff --git a/src/transformers/modeling_tf_utils.py b/src/transformers/modeling_tf_utils.py
index b562686b68..2ff8840303 100644
--- a/src/transformers/modeling_tf_utils.py
+++ b/src/transformers/modeling_tf_utils.py
@@ -102,15 +102,15 @@ def keras_serializable(cls):
1. Adding a `transformers_config` dict to the Keras config dictionary in `get_config` (called by Keras at
serialization time.
- 2. Wrapping `__init__` to accept that `transformers_config` dict (passed by Keras at deserialization
- time) and convert it to a config object for the actual layer initializer.
+ 2. Wrapping `__init__` to accept that `transformers_config` dict (passed by Keras at deserialization time) and
+ convert it to a config object for the actual layer initializer.
3. Registering the class as a custom object in Keras (if the Tensorflow version supports this), so that it does not
need to be supplied in `custom_objects` in the call to `tf.keras.models.load_model`.
Args:
cls (a `tf.keras.layers.Layers subclass`):
- Typically a `TF.MainLayer` class in this project, in general must accept a `config` argument to
- its initializer.
+ Typically a `TF.MainLayer` class in this project, in general must accept a `config` argument to its
+ initializer.
Returns:
The same class object, with modifications for Keras deserialization.
@@ -648,20 +648,20 @@ class TFPreTrainedModel(tf.keras.Model, TFModelUtilsMixin, TFGenerationMixin, Pu
r"""
Base class for all TF models.
- [`TFPreTrainedModel`] takes care of storing the configuration of the models and handles methods
- for loading, downloading and saving models as well as a few methods common to all models to:
+ [`TFPreTrainedModel`] takes care of storing the configuration of the models and handles methods for loading,
+ downloading and saving models as well as a few methods common to all models to:
- resize the input embeddings,
- prune heads in the self-attention heads.
Class attributes (overridden by derived classes):
- - **config_class** ([`PretrainedConfig`]) -- A subclass of
- [`PretrainedConfig`] to use as configuration class for this model architecture.
- - **base_model_prefix** (`str`) -- A string indicating the attribute associated to the base model in
- derived classes of the same architecture adding modules on top of the base model.
- - **main_input_name** (`str`) -- The name of the principal input to the model (often `input_ids` for
- NLP models, `pixel_values` for vision models and `input_values` for speech models).
+ - **config_class** ([`PretrainedConfig`]) -- A subclass of [`PretrainedConfig`] to use as configuration class
+ for this model architecture.
+ - **base_model_prefix** (`str`) -- A string indicating the attribute associated to the base model in derived
+ classes of the same architecture adding modules on top of the base model.
+ - **main_input_name** (`str`) -- The name of the principal input to the model (often `input_ids` for NLP
+ models, `pixel_values` for vision models and `input_values` for speech models).
"""
config_class = None
base_model_prefix = ""
@@ -1104,9 +1104,8 @@ class TFPreTrainedModel(tf.keras.Model, TFModelUtilsMixin, TFGenerationMixin, Pu
Arguments:
new_num_tokens (`int`, *optional*):
The number of new tokens in the embedding matrix. Increasing the size will add newly initialized
- vectors at the end. Reducing the size will remove vectors from the end. If not provided or `None`,
- just returns a pointer to the input tokens `tf.Variable` module of the model without doing
- anything.
+ vectors at the end. Reducing the size will remove vectors from the end. If not provided or `None`, just
+ returns a pointer to the input tokens `tf.Variable` module of the model without doing anything.
Return:
`tf.Variable`: Pointer to the input tokens Embeddings Module of the model.
@@ -1234,8 +1233,8 @@ class TFPreTrainedModel(tf.keras.Model, TFModelUtilsMixin, TFGenerationMixin, Pu
vectors from the end. If not provided or `None`, just returns None
Return:
- `tf.Variable`: Pointer to the resized decoder or None if the output embeddings are different from the
- input ones.
+ `tf.Variable`: Pointer to the resized decoder or None if the output embeddings are different from the input
+ ones.
"""
new_lm_head_decoder = old_lm_head_decoder
is_input_output_equals = tf.reduce_any(
@@ -1273,8 +1272,8 @@ class TFPreTrainedModel(tf.keras.Model, TFModelUtilsMixin, TFGenerationMixin, Pu
``tf.Variable``` module of the model without doing anything.
Return:
- `tf.Variable`: Pointer to the resized Embedding Module or the old Embedding Module if
- `new_num_tokens` is `None`
+ `tf.Variable`: Pointer to the resized Embedding Module or the old Embedding Module if `new_num_tokens` is
+ `None`
"""
old_embedding_dim = shape_list(old_embeddings)[1]
init_range = getattr(self.config, "initializer_range", 0.02)
@@ -1297,9 +1296,9 @@ class TFPreTrainedModel(tf.keras.Model, TFModelUtilsMixin, TFGenerationMixin, Pu
Arguments:
heads_to_prune (`Dict[int, List[int]]`):
- Dictionary with keys being selected layer indices (`int`) and associated values being the list of
- heads to prune in said layer (list of `int`). For instance {1: [0, 2], 2: [2, 3]} will prune heads
- 0 and 2 on layer 1 and heads 2 and 3 on layer 2.
+ Dictionary with keys being selected layer indices (`int`) and associated values being the list of heads
+ to prune in said layer (list of `int`). For instance {1: [0, 2], 2: [2, 3]} will prune heads 0 and 2 on
+ layer 1 and heads 2 and 3 on layer 2.
"""
raise NotImplementedError
@@ -1322,16 +1321,14 @@ class TFPreTrainedModel(tf.keras.Model, TFModelUtilsMixin, TFGenerationMixin, Pu
- Using `push_to_hub=True` will synchronize the repository you are pushing to with
- `save_directory`, which requires `save_directory` to be a local clone of the repo you are
- pushing to if it's an existing folder. Pass along `temp_dir=True` to use a temporary directory
- instead.
+ Using `push_to_hub=True` will synchronize the repository you are pushing to with `save_directory`,
+ which requires `save_directory` to be a local clone of the repo you are pushing to if it's an existing
+ folder. Pass along `temp_dir=True` to use a temporary directory instead.
kwargs:
- Additional key word arguments passed along to the
- [`~file_utils.PushToHubMixin.push_to_hub`] method.
+ Additional key word arguments passed along to the [`~file_utils.PushToHubMixin.push_to_hub`] method.
"""
if os.path.isfile(save_directory):
logger.error(f"Provided path ({save_directory}) should be a directory, not a file")
@@ -1378,15 +1375,14 @@ class TFPreTrainedModel(tf.keras.Model, TFModelUtilsMixin, TFGenerationMixin, Pu
Can be either:
- A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co.
- Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under
- a user or organization name, like `dbmdz/bert-base-german-cased`.
+ Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under a
+ user or organization name, like `dbmdz/bert-base-german-cased`.
- A path to a *directory* containing model weights saved using
[`~TFPreTrainedModel.save_pretrained`], e.g., `./my_model_directory/`.
- - A path or url to a *PyTorch state_dict save file* (e.g, `./pt_model/pytorch_model.bin`). In
- this case, `from_pt` should be set to `True` and a configuration object should be provided
- as `config` argument. This loading path is slower than converting the PyTorch model in a
- TensorFlow model using the provided conversion scripts and loading the TensorFlow model
- afterwards.
+ - A path or url to a *PyTorch state_dict save file* (e.g, `./pt_model/pytorch_model.bin`). In this
+ case, `from_pt` should be set to `True` and a configuration object should be provided as `config`
+ argument. This loading path is slower than converting the PyTorch model in a TensorFlow model
+ using the provided conversion scripts and loading the TensorFlow model afterwards.
- `None` if you are both providing the configuration and state dictionary (resp. with keyword
arguments `config` and `state_dict`).
model_args (sequence of positional arguments, *optional*):
@@ -1402,8 +1398,8 @@ class TFPreTrainedModel(tf.keras.Model, TFModelUtilsMixin, TFGenerationMixin, Pu
- The model is a model provided by the library (loaded with the *model id* string of a pretrained
model).
- - The model was saved using [`~TFPreTrainedModel.save_pretrained`] and is reloaded
- by supplying the save directory.
+ - The model was saved using [`~TFPreTrainedModel.save_pretrained`] and is reloaded by supplying the
+ save directory.
- The model is loaded by supplying a local directory as `pretrained_model_name_or_path` and a
configuration JSON file named *config.json* is found in the directory.
from_pt: (`bool`, *optional*, defaults to `False`):
@@ -1422,14 +1418,16 @@ class TFPreTrainedModel(tf.keras.Model, TFModelUtilsMixin, TFGenerationMixin, Pu
resume_download (`bool`, *optional*, defaults to `False`):
Whether or not to delete incompletely received files. Will attempt to resume the download if such a
file exists.
- proxies: (`Dict[str, str], `optional`): A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',
- 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. output_loading_info(`bool`, *optional*, defaults to `False`):
- Whether ot not to also return a dictionary containing missing keys, unexpected keys and error messages.
+ proxies:
+ (`Dict[str, str], `optional`): A dictionary of proxy servers to use by protocol or endpoint, e.g.,
+ `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request.
+ output_loading_info(`bool`, *optional*, defaults to `False`): Whether ot not to also return a
+ dictionary containing missing keys, unexpected keys and error messages.
local_files_only(`bool`, *optional*, defaults to `False`):
Whether or not to only look at local files (e.g., not try doanloading the model).
use_auth_token (`str` or *bool*, *optional*):
- The token to use as HTTP bearer authorization for remote files. If `True`, will use the token
- generated when running `transformers-cli login` (stored in `~/.huggingface`).
+ The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated
+ when running `transformers-cli login` (stored in `~/.huggingface`).
revision(`str`, *optional*, defaults to `"main"`):
The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a
git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any
@@ -1447,10 +1445,10 @@ class TFPreTrainedModel(tf.keras.Model, TFModelUtilsMixin, TFGenerationMixin, Pu
underlying model's `__init__` method (we assume all relevant updates to the configuration have
already been done)
- If a configuration is not provided, `kwargs` will be first passed to the configuration class
- initialization function ([`~PretrainedConfig.from_pretrained`]). Each key of
- `kwargs` that corresponds to a configuration attribute will be used to override said attribute
- with the supplied `kwargs` value. Remaining keys that do not correspond to any configuration
- attribute will be passed to the underlying model's `__init__` function.
+ initialization function ([`~PretrainedConfig.from_pretrained`]). Each key of `kwargs` that
+ corresponds to a configuration attribute will be used to override said attribute with the
+ supplied `kwargs` value. Remaining keys that do not correspond to any configuration attribute
+ will be passed to the underlying model's `__init__` function.
@@ -1782,19 +1780,20 @@ class TFSharedEmbeddings(tf.keras.layers.Layer):
In linear mode, should be a float tensor with shape `[batch_size, length, hidden_size]`.
mode (`str`, defaults to `"embedding"`):
- A valid value is either `"embedding"` or `"linear"`, the first one indicates that the layer
- should be used as an embedding layer, the second one that the layer should be used as a linear decoder.
+ A valid value is either `"embedding"` or `"linear"`, the first one indicates that the layer should be
+ used as an embedding layer, the second one that the layer should be used as a linear decoder.
Returns:
- `tf.Tensor`: In embedding mode, the output is a float32 embedding tensor, with shape
- `[batch_size, length, embedding_size]`.
+ `tf.Tensor`: In embedding mode, the output is a float32 embedding tensor, with shape `[batch_size, length,
+ embedding_size]`.
In linear mode, the output is a float32 with shape `[batch_size, length, vocab_size]`.
Raises:
ValueError: if `mode` is not valid.
- Shared weights logic is adapted from [here](https://github.com/tensorflow/models/blob/a009f4fb9d2fc4949e32192a944688925ef78659/official/transformer/v2/embedding_layer.py#L24).
+ Shared weights logic is adapted from
+ [here](https://github.com/tensorflow/models/blob/a009f4fb9d2fc4949e32192a944688925ef78659/official/transformer/v2/embedding_layer.py#L24).
"""
if mode == "embedding":
return self._embedding(inputs)
@@ -1842,14 +1841,12 @@ class TFSequenceSummary(tf.keras.layers.Layer):
- `"attn"` -- Not implemented now, use multi-head attention
- **summary_use_proj** (`bool`) -- Add a projection after the vector extraction.
- - **summary_proj_to_labels** (`bool`) -- If `True`, the projection outputs to
- `config.num_labels` classes (otherwise to `config.hidden_size`).
- - **summary_activation** (`Optional[str]`) -- Set to `"tanh"` to add a tanh activation to the
- output, another string or `None` will add no activation.
- - **summary_first_dropout** (`float`) -- Optional dropout probability before the projection and
- activation.
- - **summary_last_dropout** (`float`)-- Optional dropout probability after the projection and
- activation.
+ - **summary_proj_to_labels** (`bool`) -- If `True`, the projection outputs to `config.num_labels` classes
+ (otherwise to `config.hidden_size`).
+ - **summary_activation** (`Optional[str]`) -- Set to `"tanh"` to add a tanh activation to the output,
+ another string or `None` will add no activation.
+ - **summary_first_dropout** (`float`) -- Optional dropout probability before the projection and activation.
+ - **summary_last_dropout** (`float`)-- Optional dropout probability after the projection and activation.
initializer_range (`float`, defaults to 0.02): The standard deviation to use to initialize the weights.
kwargs:
diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py
index e4d58ff501..34f9f857ec 100644
--- a/src/transformers/modeling_utils.py
+++ b/src/transformers/modeling_utils.py
@@ -176,8 +176,8 @@ class ModuleUtilsMixin:
"""
Add a memory hook before and after each sub-module forward pass to record increase in memory consumption.
- Increase in memory consumption is stored in a `mem_rss_diff` attribute for each module and can be reset to
- zero with `model.reset_memory_hooks_state()`.
+ Increase in memory consumption is stored in a `mem_rss_diff` attribute for each module and can be reset to zero
+ with `model.reset_memory_hooks_state()`.
"""
for module in self.modules():
module.register_forward_pre_hook(self._hook_rss_memory_pre_forward)
@@ -186,8 +186,7 @@ class ModuleUtilsMixin:
def reset_memory_hooks_state(self):
"""
- Reset the `mem_rss_diff` attribute of each module (see
- [`~modeling_utils.ModuleUtilsMixin.add_memory_hooks`]).
+ Reset the `mem_rss_diff` attribute of each module (see [`~modeling_utils.ModuleUtilsMixin.add_memory_hooks`]).
"""
for module in self.modules():
module.mem_rss_diff = 0
@@ -316,8 +315,8 @@ class ModuleUtilsMixin:
Whether or not the attentions scores are computed by chunks or not.
Returns:
- `torch.Tensor` with shape `[num_hidden_layers x batch x num_heads x seq_length x seq_length]` or
- list with `[None]` for each layer.
+ `torch.Tensor` with shape `[num_hidden_layers x batch x num_heads x seq_length x seq_length]` or list with
+ `[None]` for each layer.
"""
if head_mask is not None:
head_mask = self._convert_head_mask_to_5d(head_mask, num_hidden_layers)
@@ -389,7 +388,8 @@ class ModuleUtilsMixin:
"""
Get number of (optionally, non-embeddings) floating-point operations for the forward and backward passes of a
batch with this transformer model. Default approximation neglects the quadratic dependency on the number of
- tokens (valid if `12 * d_model << sequence_length`) as laid out in [this paper](https://arxiv.org/pdf/2001.08361.pdf) section 2.1. Should be overridden for transformers with parameter
+ tokens (valid if `12 * d_model << sequence_length`) as laid out in [this
+ paper](https://arxiv.org/pdf/2001.08361.pdf) section 2.1. Should be overridden for transformers with parameter
re-use e.g. Albert or Universal Transformers, or if doing long-range modeling with very high sequence lengths.
Args:
@@ -413,30 +413,28 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin, GenerationMixin, PushToHubMix
r"""
Base class for all models.
- [`PreTrainedModel`] takes care of storing the configuration of the models and handles methods
- for loading, downloading and saving models as well as a few methods common to all models to:
+ [`PreTrainedModel`] takes care of storing the configuration of the models and handles methods for loading,
+ downloading and saving models as well as a few methods common to all models to:
- resize the input embeddings,
- prune heads in the self-attention heads.
Class attributes (overridden by derived classes):
- - **config_class** ([`PretrainedConfig`]) -- A subclass of
- [`PretrainedConfig`] to use as configuration class for this model architecture.
- - **load_tf_weights** (`Callable`) -- A python *method* for loading a TensorFlow checkpoint in a PyTorch
- model, taking as arguments:
+ - **config_class** ([`PretrainedConfig`]) -- A subclass of [`PretrainedConfig`] to use as configuration class
+ for this model architecture.
+ - **load_tf_weights** (`Callable`) -- A python *method* for loading a TensorFlow checkpoint in a PyTorch model,
+ taking as arguments:
- - **model** ([`PreTrainedModel`]) -- An instance of the model on which to load the
- TensorFlow checkpoint.
- - **config** ([`PreTrainedConfig`]) -- An instance of the configuration associated to
- the model.
+ - **model** ([`PreTrainedModel`]) -- An instance of the model on which to load the TensorFlow checkpoint.
+ - **config** ([`PreTrainedConfig`]) -- An instance of the configuration associated to the model.
- **path** (`str`) -- A path to the TensorFlow checkpoint.
- - **base_model_prefix** (`str`) -- A string indicating the attribute associated to the base model in
- derived classes of the same architecture adding modules on top of the base model.
+ - **base_model_prefix** (`str`) -- A string indicating the attribute associated to the base model in derived
+ classes of the same architecture adding modules on top of the base model.
- **is_parallelizable** (`bool`) -- A flag indicating whether this model supports model parallelization.
- - **main_input_name** (`str`) -- The name of the principal input to the model (often `input_ids` for
- NLP models, `pixel_values` for vision models and `input_values` for speech models).
+ - **main_input_name** (`str`) -- The name of the principal input to the model (often `input_ids` for NLP
+ models, `pixel_values` for vision models and `input_values` for speech models).
"""
config_class = None
base_model_prefix = ""
@@ -539,8 +537,8 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin, GenerationMixin, PushToHubMix
a floating dtype to set to.
Returns:
- `torch.dtype`: the original `dtype` that can be used to restore `torch.set_default_dtype(dtype)`
- if it was modified. If it wasn't, returns `None`.
+ `torch.dtype`: the original `dtype` that can be used to restore `torch.set_default_dtype(dtype)` if it was
+ modified. If it wasn't, returns `None`.
Note `set_default_dtype` currently only works with floating-point types and asserts if for example,
`torch.int64` is passed. So if a non-float `dtype` is passed this functions will throw an exception.
@@ -607,8 +605,8 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin, GenerationMixin, PushToHubMix
"""
Tie the weights between the input embeddings and the output embeddings.
- If the `torchscript` flag is set in the configuration, can't handle parameter sharing so we are cloning
- the weights instead.
+ If the `torchscript` flag is set in the configuration, can't handle parameter sharing so we are cloning the
+ weights instead.
"""
output_embeddings = self.get_output_embeddings()
if output_embeddings is not None and self.config.tie_word_embeddings:
@@ -725,9 +723,8 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin, GenerationMixin, PushToHubMix
Arguments:
new_num_tokens (`int`, *optional*):
The number of new tokens in the embedding matrix. Increasing the size will add newly initialized
- vectors at the end. Reducing the size will remove vectors from the end. If not provided or `None`,
- just returns a pointer to the input tokens `torch.nn.Embedding` module of the model without doing
- anything.
+ vectors at the end. Reducing the size will remove vectors from the end. If not provided or `None`, just
+ returns a pointer to the input tokens `torch.nn.Embedding` module of the model without doing anything.
Return:
`torch.nn.Embedding`: Pointer to the input tokens Embeddings Module of the model.
@@ -836,12 +833,13 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin, GenerationMixin, PushToHubMix
Increasing the size will add newly initialized vectors at the end. Reducing the size will remove
vectors from the end. If not provided or `None`, just returns a pointer to the input tokens
- ``torch.nn.Linear``` module of the model without doing anything. transposed (`bool`, *optional*, defaults to `False`): Whether `old_lm_head` is transposed or not. If True `old_lm_head.size()` is `lm_head_dim,
- vocab_size` else `vocab_size, lm_head_dim`.
+ ``torch.nn.Linear``` module of the model without doing anything. transposed (`bool`, *optional*,
+ defaults to `False`): Whether `old_lm_head` is transposed or not. If True `old_lm_head.size()` is
+ `lm_head_dim, vocab_size` else `vocab_size, lm_head_dim`.
Return:
- `torch.nn.Linear`: Pointer to the resized Linear Module or the old Linear Module if
- `new_num_tokens` is `None`
+ `torch.nn.Linear`: Pointer to the resized Linear Module or the old Linear Module if `new_num_tokens` is
+ `None`
"""
if new_num_tokens is None:
return old_lm_head
@@ -944,9 +942,9 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin, GenerationMixin, PushToHubMix
Arguments:
heads_to_prune (`Dict[int, List[int]]`):
- Dictionary with keys being selected layer indices (`int`) and associated values being the list of
- heads to prune in said layer (list of `int`). For instance {1: [0, 2], 2: [2, 3]} will prune heads
- 0 and 2 on layer 1 and heads 2 and 3 on layer 2.
+ Dictionary with keys being selected layer indices (`int`) and associated values being the list of heads
+ to prune in said layer (list of `int`). For instance {1: [0, 2], 2: [2, 3]} will prune heads 0 and 2 on
+ layer 1 and heads 2 and 3 on layer 2.
"""
# save new sets of pruned heads as union of previously stored pruned heads and newly pruned heads
for layer, heads in heads_to_prune.items():
@@ -1004,12 +1002,12 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin, GenerationMixin, PushToHubMix
Directory to which to save. Will be created if it doesn't exist.
save_config (`bool`, *optional*, defaults to `True`):
Whether or not to save the config of the model. Useful when in distributed training like TPUs and need
- to call this function on all processes. In this case, set `save_config=True` only on the main
- process to avoid race conditions.
+ to call this function on all processes. In this case, set `save_config=True` only on the main process
+ to avoid race conditions.
state_dict (nested dictionary of `torch.Tensor`):
- The state dictionary of the model to save. Will default to `self.state_dict()`, but can be used to
- only save parts of the model or if special precautions need to be taken when recovering the state
- dictionary of a model (like when using model parallelism).
+ The state dictionary of the model to save. Will default to `self.state_dict()`, but can be used to only
+ save parts of the model or if special precautions need to be taken when recovering the state dictionary
+ of a model (like when using model parallelism).
save_function (`Callable`):
The function to use to save the state dictionary. Useful on distributed training like TPUs when one
need to replace `torch.save` by another method.
@@ -1018,16 +1016,14 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin, GenerationMixin, PushToHubMix
- Using `push_to_hub=True` will synchronize the repository you are pushing to with
- `save_directory`, which requires `save_directory` to be a local clone of the repo you are
- pushing to if it's an existing folder. Pass along `temp_dir=True` to use a temporary directory
- instead.
+ Using `push_to_hub=True` will synchronize the repository you are pushing to with `save_directory`,
+ which requires `save_directory` to be a local clone of the repo you are pushing to if it's an existing
+ folder. Pass along `temp_dir=True` to use a temporary directory instead.
kwargs:
- Additional key word arguments passed along to the
- [`~file_utils.PushToHubMixin.push_to_hub`] method.
+ Additional key word arguments passed along to the [`~file_utils.PushToHubMixin.push_to_hub`] method.
"""
if os.path.isfile(save_directory):
logger.error(f"Provided path ({save_directory}) should be a directory, not a file")
@@ -1079,8 +1075,8 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin, GenerationMixin, PushToHubMix
r"""
Instantiate a pretrained pytorch model from a pre-trained model configuration.
- The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated). To
- train the model, you should first set it back in training mode with `model.train()`.
+ The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated). To train
+ the model, you should first set it back in training mode with `model.train()`.
The warning *Weights from XXX not initialized from pretrained model* means that the weights of XXX do not come
pretrained with the rest of the model. It is up to you to train those weights with a downstream fine-tuning
@@ -1094,17 +1090,17 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin, GenerationMixin, PushToHubMix
Can be either:
- A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co.
- Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under
- a user or organization name, like `dbmdz/bert-base-german-cased`.
+ Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under a
+ user or organization name, like `dbmdz/bert-base-german-cased`.
- A path to a *directory* containing model weights saved using
[`~PreTrainedModel.save_pretrained`], e.g., `./my_model_directory/`.
- A path or url to a *tensorflow index checkpoint file* (e.g, `./tf_model/model.ckpt.index`). In
- this case, `from_tf` should be set to `True` and a configuration object should be provided
- as `config` argument. This loading path is slower than converting the TensorFlow checkpoint in
- a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards.
+ this case, `from_tf` should be set to `True` and a configuration object should be provided as
+ `config` argument. This loading path is slower than converting the TensorFlow checkpoint in a
+ PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards.
- A path or url to a model folder containing a *flax checkpoint file* in *.msgpack* format (e.g,
- `./flax_model/` containing `flax_model.msgpack`). In this case, `from_flax` should be set
- to `True`.
+ `./flax_model/` containing `flax_model.msgpack`). In this case, `from_flax` should be set to
+ `True`.
- `None` if you are both providing the configuration and state dictionary (resp. with keyword
arguments `config` and `state_dict`).
model_args (sequence of positional arguments, *optional*):
@@ -1120,16 +1116,15 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin, GenerationMixin, PushToHubMix
- The model is a model provided by the library (loaded with the *model id* string of a pretrained
model).
- - The model was saved using [`~PreTrainedModel.save_pretrained`] and is reloaded
- by supplying the save directory.
+ - The model was saved using [`~PreTrainedModel.save_pretrained`] and is reloaded by supplying the
+ save directory.
- The model is loaded by supplying a local directory as `pretrained_model_name_or_path` and a
configuration JSON file named *config.json* is found in the directory.
state_dict (`Dict[str, torch.Tensor]`, *optional*):
A state dictionary to use instead of a state dictionary loaded from saved weights file.
This option can be used if you want to create a model from a pretrained configuration but load your own
- weights. In this case though, you should check if using
- [`~PreTrainedModel.save_pretrained`] and
+ weights. In this case though, you should check if using [`~PreTrainedModel.save_pretrained`] and
[`~PreTrainedModel.from_pretrained`] is not a simpler option.
cache_dir (`Union[str, os.PathLike]`, *optional*):
Path to a directory in which a downloaded pretrained model configuration should be cached if the
@@ -1151,14 +1146,15 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin, GenerationMixin, PushToHubMix
Whether or not to delete incompletely received files. Will attempt to resume the download if such a
file exists.
proxies (`Dict[str, str]`, *optional*):
- A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request.
+ A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',
+ 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request.
output_loading_info(`bool`, *optional*, defaults to `False`):
Whether ot not to also return a dictionary containing missing keys, unexpected keys and error messages.
local_files_only(`bool`, *optional*, defaults to `False`):
Whether or not to only look at local files (i.e., do not try to download the model).
use_auth_token (`str` or *bool*, *optional*):
- The token to use as HTTP bearer authorization for remote files. If `True`, will use the token
- generated when running `transformers-cli login` (stored in `~/.huggingface`).
+ The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated
+ when running `transformers-cli login` (stored in `~/.huggingface`).
revision(`str`, *optional*, defaults to `"main"`):
The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a
git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any
@@ -1173,14 +1169,14 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin, GenerationMixin, PushToHubMix
Tries to not use more than 1x model size in CPU memory (including peak memory) while loading the model.
This is an experimental feature and a subject to change at any moment.
torch_dtype (`str` or `torch.dtype`, *optional*):
- Override the default `torch.dtype` and load the model under this dtype. If `"auto"` is passed the
- dtype will be automatically derived from the model's weights.
+ Override the default `torch.dtype` and load the model under this dtype. If `"auto"` is passed the dtype
+ will be automatically derived from the model's weights.
- One should only disable *_fast_init* to ensure backwards compatibility with
- `transformers.__version__ < 4.6.0` for seeded model initialization. This argument will be removed
- at the next major version. See [pull request 11471](https://github.com/huggingface/transformers/pull/11471) for more information.
+ One should only disable *_fast_init* to ensure backwards compatibility with `transformers.__version__ <
+ 4.6.0` for seeded model initialization. This argument will be removed at the next major version. See
+ [pull request 11471](https://github.com/huggingface/transformers/pull/11471) for more information.
@@ -1193,10 +1189,10 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin, GenerationMixin, PushToHubMix
underlying model's `__init__` method (we assume all relevant updates to the configuration have
already been done)
- If a configuration is not provided, `kwargs` will be first passed to the configuration class
- initialization function ([`~PretrainedConfig.from_pretrained`]). Each key of
- `kwargs` that corresponds to a configuration attribute will be used to override said attribute
- with the supplied `kwargs` value. Remaining keys that do not correspond to any configuration
- attribute will be passed to the underlying model's `__init__` function.
+ initialization function ([`~PretrainedConfig.from_pretrained`]). Each key of `kwargs` that
+ corresponds to a configuration attribute will be used to override said attribute with the
+ supplied `kwargs` value. Remaining keys that do not correspond to any configuration attribute
+ will be passed to the underlying model's `__init__` function.
@@ -1206,8 +1202,8 @@ class PreTrainedModel(nn.Module, ModuleUtilsMixin, GenerationMixin, PushToHubMix
- Activate the special ["offline-mode"](https://huggingface.co/transformers/installation.html#offline-mode) to use this method in a firewalled
- environment.
+ Activate the special ["offline-mode"](https://huggingface.co/transformers/installation.html#offline-mode) to
+ use this method in a firewalled environment.
@@ -1812,8 +1808,8 @@ class PoolerEndLogits(nn.Module):
Args:
config ([`PretrainedConfig`]):
- The config used by the model, will be used to grab the `hidden_size` of the model and the
- `layer_norm_eps` to use.
+ The config used by the model, will be used to grab the `hidden_size` of the model and the `layer_norm_eps`
+ to use.
"""
def __init__(self, config: PretrainedConfig):
@@ -1844,8 +1840,8 @@ class PoolerEndLogits(nn.Module):
- One of `start_states` or `start_positions` should be not obj:*None*. If both are set,
- `start_positions` overrides `start_states`.
+ One of `start_states` or `start_positions` should be not obj:*None*. If both are set, `start_positions`
+ overrides `start_states`.
@@ -1910,8 +1906,8 @@ class PoolerAnswerClass(nn.Module):
- One of `start_states` or `start_positions` should be not obj:*None*. If both are set,
- `start_positions` overrides `start_states`.
+ One of `start_states` or `start_positions` should be not obj:*None*. If both are set, `start_positions`
+ overrides `start_states`.
@@ -1977,8 +1973,8 @@ class SQuADHead(nn.Module):
Args:
config ([`PretrainedConfig`]):
- The config used by the model, will be used to grab the `hidden_size` of the model and the
- `layer_norm_eps` to use.
+ The config used by the model, will be used to grab the `hidden_size` of the model and the `layer_norm_eps`
+ to use.
"""
def __init__(self, config):
@@ -2106,14 +2102,12 @@ class SequenceSummary(nn.Module):
- `"attn"` -- Not implemented now, use multi-head attention
- **summary_use_proj** (`bool`) -- Add a projection after the vector extraction.
- - **summary_proj_to_labels** (`bool`) -- If `True`, the projection outputs to
- `config.num_labels` classes (otherwise to `config.hidden_size`).
- - **summary_activation** (`Optional[str]`) -- Set to `"tanh"` to add a tanh activation to the
- output, another string or `None` will add no activation.
- - **summary_first_dropout** (`float`) -- Optional dropout probability before the projection and
- activation.
- - **summary_last_dropout** (`float`)-- Optional dropout probability after the projection and
- activation.
+ - **summary_proj_to_labels** (`bool`) -- If `True`, the projection outputs to `config.num_labels` classes
+ (otherwise to `config.hidden_size`).
+ - **summary_activation** (`Optional[str]`) -- Set to `"tanh"` to add a tanh activation to the output,
+ another string or `None` will add no activation.
+ - **summary_first_dropout** (`float`) -- Optional dropout probability before the projection and activation.
+ - **summary_last_dropout** (`float`)-- Optional dropout probability after the projection and activation.
"""
def __init__(self, config: PretrainedConfig):
@@ -2155,8 +2149,7 @@ class SequenceSummary(nn.Module):
hidden_states (`torch.FloatTensor` of shape `[batch_size, seq_len, hidden_size]`):
The hidden states of the last layer.
cls_index (`torch.LongTensor` of shape `[batch_size]` or `[batch_size, ...]` where ... are optional leading dimensions of `hidden_states`, *optional*):
- Used if `summary_type == "cls_index"` and takes the last token of the sequence as classification
- token.
+ Used if `summary_type == "cls_index"` and takes the last token of the sequence as classification token.
Returns:
`torch.FloatTensor`: The summary of the sequence hidden states.
@@ -2285,8 +2278,7 @@ def prune_layer(
dim (`int`, *optional*): The dimension on which to keep the indices.
Returns:
- `torch.nn.Linear` or [`~modeling_utils.Conv1D`]: The pruned layer as a new layer with
- `requires_grad=True`.
+ `torch.nn.Linear` or [`~modeling_utils.Conv1D`]: The pruned layer as a new layer with `requires_grad=True`.
"""
if isinstance(layer, nn.Linear):
return prune_linear_layer(layer, index, dim=0 if dim is None else dim)
@@ -2300,11 +2292,11 @@ def apply_chunking_to_forward(
forward_fn: Callable[..., torch.Tensor], chunk_size: int, chunk_dim: int, *input_tensors
) -> torch.Tensor:
"""
- This function chunks the `input_tensors` into smaller input tensor parts of size `chunk_size` over the
- dimension `chunk_dim`. It then applies a layer `forward_fn` to each chunk independently to save memory.
+ This function chunks the `input_tensors` into smaller input tensor parts of size `chunk_size` over the dimension
+ `chunk_dim`. It then applies a layer `forward_fn` to each chunk independently to save memory.
- If the `forward_fn` is independent across the `chunk_dim` this function will yield the same result as
- directly applying `forward_fn` to `input_tensors`.
+ If the `forward_fn` is independent across the `chunk_dim` this function will yield the same result as directly
+ applying `forward_fn` to `input_tensors`.
Args:
forward_fn (`Callable[..., torch.Tensor]`):
diff --git a/src/transformers/models/albert/configuration_albert.py b/src/transformers/models/albert/configuration_albert.py
index 1bd0aa786b..fb87e845d2 100644
--- a/src/transformers/models/albert/configuration_albert.py
+++ b/src/transformers/models/albert/configuration_albert.py
@@ -13,7 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" ALBERT model configuration """
+""" ALBERT model configuration"""
from collections import OrderedDict
from typing import Mapping
@@ -35,19 +35,18 @@ ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class AlbertConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`AlbertModel`] or a
- [`TFAlbertModel`]. It is used to instantiate an ALBERT model according to the specified
- arguments, defining the model architecture. Instantiating a configuration with the defaults will yield a similar
- configuration to that of the ALBERT [xxlarge](https://huggingface.co/albert-xxlarge-v2) architecture.
+ This is the configuration class to store the configuration of a [`AlbertModel`] or a [`TFAlbertModel`]. It is used
+ to instantiate an ALBERT model according to the specified arguments, defining the model architecture. Instantiating
+ a configuration with the defaults will yield a similar configuration to that of the ALBERT
+ [xxlarge](https://huggingface.co/albert-xxlarge-v2) architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
vocab_size (`int`, *optional*, defaults to 30000):
Vocabulary size of the ALBERT model. Defines the number of different tokens that can be represented by the
- `inputs_ids` passed when calling [`AlbertModel`] or
- [`TFAlbertModel`].
+ `inputs_ids` passed when calling [`AlbertModel`] or [`TFAlbertModel`].
embedding_size (`int`, *optional*, defaults to 128):
Dimensionality of vocabulary embeddings.
hidden_size (`int`, *optional*, defaults to 4096):
@@ -63,8 +62,8 @@ class AlbertConfig(PretrainedConfig):
inner_group_num (`int`, *optional*, defaults to 1):
The number of inner repetition of attention and ffn.
hidden_act (`str` or `Callable`, *optional*, defaults to `"gelu_new"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"silu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"silu"` and `"gelu_new"` are supported.
hidden_dropout_prob (`float`, *optional*, defaults to 0):
The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
attention_probs_dropout_prob (`float`, *optional*, defaults to 0):
@@ -73,8 +72,7 @@ class AlbertConfig(PretrainedConfig):
The maximum sequence length that this model might ever be used with. Typically set this to something large
(e.g., 512 or 1024 or 2048).
type_vocab_size (`int`, *optional*, defaults to 2):
- The vocabulary size of the `token_type_ids` passed when calling [`AlbertModel`] or
- [`TFAlbertModel`].
+ The vocabulary size of the `token_type_ids` passed when calling [`AlbertModel`] or [`TFAlbertModel`].
initializer_range (`float`, *optional*, defaults to 0.02):
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
layer_norm_eps (`float`, *optional*, defaults to 1e-12):
@@ -82,10 +80,11 @@ class AlbertConfig(PretrainedConfig):
classifier_dropout_prob (`float`, *optional*, defaults to 0.1):
The dropout ratio for attached classifiers.
position_embedding_type (`str`, *optional*, defaults to `"absolute"`):
- Type of position embedding. Choose one of `"absolute"`, `"relative_key"`,
- `"relative_key_query"`. For positional embeddings use `"absolute"`. For more information on
- `"relative_key"`, please refer to [Self-Attention with Relative Position Representations (Shaw et al.)](https://arxiv.org/abs/1803.02155). For more information on `"relative_key_query"`, please refer to
- *Method 4* in [Improve Transformer Models with Better Relative Position Embeddings (Huang et al.)](https://arxiv.org/abs/2009.13658).
+ Type of position embedding. Choose one of `"absolute"`, `"relative_key"`, `"relative_key_query"`. For
+ positional embeddings use `"absolute"`. For more information on `"relative_key"`, please refer to
+ [Self-Attention with Relative Position Representations (Shaw et al.)](https://arxiv.org/abs/1803.02155).
+ For more information on `"relative_key_query"`, please refer to *Method 4* in [Improve Transformer Models
+ with Better Relative Position Embeddings (Huang et al.)](https://arxiv.org/abs/2009.13658).
Examples:
diff --git a/src/transformers/models/albert/modeling_albert.py b/src/transformers/models/albert/modeling_albert.py
index a7771852b4..235351279a 100755
--- a/src/transformers/models/albert/modeling_albert.py
+++ b/src/transformers/models/albert/modeling_albert.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-"""PyTorch ALBERT model. """
+"""PyTorch ALBERT model."""
import math
import os
@@ -529,12 +529,13 @@ class AlbertForPreTrainingOutput(ModelOutput):
Prediction scores of the next sequence prediction (classification) head (scores of True/False continuation
before SoftMax).
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -549,19 +550,18 @@ class AlbertForPreTrainingOutput(ModelOutput):
ALBERT_START_DOCSTRING = r"""
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
Args:
config ([`AlbertConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
ALBERT_INPUTS_DOCSTRING = r"""
@@ -569,9 +569,8 @@ ALBERT_INPUTS_DOCSTRING = r"""
input_ids (`torch.LongTensor` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`AlbertTokenizer`]. See
- [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for
- details.
+ Indices can be obtained using [`AlbertTokenizer`]. See [`PreTrainedTokenizer.__call__`] and
+ [`PreTrainedTokenizer.encode`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
@@ -582,14 +581,16 @@ ALBERT_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
@@ -599,9 +600,9 @@ ALBERT_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -785,12 +786,13 @@ class AlbertForPreTraining(AlbertPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
sentence_order_label (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair
- (see `input_ids` docstring) Indices should be in `[0, 1]`. `0` indicates original order (sequence
- A, then sequence B), `1` indicates switched order (sequence B, then sequence A).
+ (see `input_ids` docstring) Indices should be in `[0, 1]`. `0` indicates original order (sequence A, then
+ sequence B), `1` indicates switched order (sequence B, then sequence A).
Returns:
@@ -935,8 +937,9 @@ class AlbertForMaskedLM(AlbertPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1014,8 +1017,9 @@ class AlbertForSequenceClassification(AlbertPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1212,12 +1216,12 @@ class AlbertForQuestionAnswering(AlbertPreTrainedModel):
r"""
start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the start of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the end of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1310,7 +1314,8 @@ class AlbertForMultipleChoice(AlbertPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where *num_choices* is the size of the second dimension of the input tensors. (see
+ Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
+ num_choices-1]` where *num_choices* is the size of the second dimension of the input tensors. (see
*input_ids* above)
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
diff --git a/src/transformers/models/albert/modeling_flax_albert.py b/src/transformers/models/albert/modeling_flax_albert.py
index c7ae117a1b..57e953a77a 100644
--- a/src/transformers/models/albert/modeling_flax_albert.py
+++ b/src/transformers/models/albert/modeling_flax_albert.py
@@ -65,12 +65,13 @@ class FlaxAlbertForPreTrainingOutput(ModelOutput):
Prediction scores of the next sequence prediction (classification) head (scores of True/False continuation
before SoftMax).
hidden_states (`tuple(jnp.ndarray)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(jnp.ndarray)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -84,12 +85,12 @@ class FlaxAlbertForPreTrainingOutput(ModelOutput):
ALBERT_START_DOCSTRING = r"""
- This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading, saving and converting weights from
- PyTorch models)
+ This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading, saving and converting weights from PyTorch models)
- This model is also a Flax Linen [flax.linen.Module](https://flax.readthedocs.io/en/latest/flax.linen.html#module) subclass. Use it as a regular Flax linen Module
- and refer to the Flax documentation for all matter related to general usage and behavior.
+ This model is also a Flax Linen [flax.linen.Module](https://flax.readthedocs.io/en/latest/flax.linen.html#module)
+ subclass. Use it as a regular Flax linen Module and refer to the Flax documentation for all matter related to
+ general usage and behavior.
Finally, this model supports inherent JAX features such as:
@@ -101,11 +102,10 @@ ALBERT_START_DOCSTRING = r"""
Parameters:
config ([`AlbertConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the
- model weights.
+ configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the model weights.
dtype (`jax.numpy.dtype`, *optional*, defaults to `jax.numpy.float32`):
- The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on
- GPUs) and `jax.numpy.bfloat16` (on TPUs).
+ The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on GPUs) and
+ `jax.numpy.bfloat16` (on TPUs).
This can be used to enable mixed-precision training or half-precision inference on GPUs or TPUs. If
specified all the computation will be performed with the given `dtype`.
@@ -113,8 +113,8 @@ ALBERT_START_DOCSTRING = r"""
**Note that this only specifies the dtype of the computation and does not influence the dtype of model
parameters.**
- If you wish to change the dtype of the model parameters, see
- [`~FlaxPreTrainedModel.to_fp16`] and [`~FlaxPreTrainedModel.to_bf16`].
+ If you wish to change the dtype of the model parameters, see [`~FlaxPreTrainedModel.to_fp16`] and
+ [`~FlaxPreTrainedModel.to_bf16`].
"""
ALBERT_INPUTS_DOCSTRING = r"""
@@ -122,9 +122,8 @@ ALBERT_INPUTS_DOCSTRING = r"""
input_ids (`numpy.ndarray` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`AlbertTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`AlbertTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`numpy.ndarray` of shape `({0})`, *optional*):
@@ -135,14 +134,16 @@ ALBERT_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`numpy.ndarray` of shape `({0})`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`numpy.ndarray` of shape `({0})`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
return_dict (`bool`, *optional*):
Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple.
@@ -817,7 +818,7 @@ class FlaxAlbertForMaskedLMModule(nn.Module):
)
-@add_start_docstrings("""Albert Model with a `language modeling` head on top. """, ALBERT_START_DOCSTRING)
+@add_start_docstrings("""Albert Model with a `language modeling` head on top.""", ALBERT_START_DOCSTRING)
class FlaxAlbertForMaskedLM(FlaxAlbertPreTrainedModel):
module_class = FlaxAlbertForMaskedLMModule
diff --git a/src/transformers/models/albert/modeling_tf_albert.py b/src/transformers/models/albert/modeling_tf_albert.py
index 05c755188f..86f6698c9f 100644
--- a/src/transformers/models/albert/modeling_tf_albert.py
+++ b/src/transformers/models/albert/modeling_tf_albert.py
@@ -13,7 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" TF 2.0 ALBERT model. """
+""" TF 2.0 ALBERT model."""
import math
from dataclasses import dataclass
@@ -657,12 +657,13 @@ class TFAlbertForPreTrainingOutput(ModelOutput):
Prediction scores of the next sequence prediction (classification) head (scores of True/False continuation
before SoftMax).
hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -677,13 +678,13 @@ class TFAlbertForPreTrainingOutput(ModelOutput):
ALBERT_START_DOCSTRING = r"""
- This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading or saving, resizing the input
- embeddings, pruning heads etc.)
+ This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use
- it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage
- and behavior.
+ This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it
+ as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and
+ behavior.
@@ -692,11 +693,11 @@ ALBERT_START_DOCSTRING = r"""
- having all inputs as keyword arguments (like PyTorch models), or
- having all inputs as a list, tuple or dict in the first positional arguments.
- This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all
- the tensors in the first argument of the model call function: `model(inputs)`.
+ This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all the
+ tensors in the first argument of the model call function: `model(inputs)`.
- If you choose this second option, there are three possibilities you can use to gather all the input Tensors in
- the first positional argument :
+ If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the
+ first positional argument :
- a single Tensor with `input_ids` only and nothing else: `model(inputs_ids)`
- a list of varying length with one or several input Tensors IN THE ORDER given in the docstring:
@@ -709,8 +710,7 @@ ALBERT_START_DOCSTRING = r"""
Args:
config ([`AlbertConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
ALBERT_INPUTS_DOCSTRING = r"""
@@ -718,9 +718,8 @@ ALBERT_INPUTS_DOCSTRING = r"""
input_ids (`Numpy array` or `tf.Tensor` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`AlbertTokenizer`]. See
- [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for
- details.
+ Indices can be obtained using [`AlbertTokenizer`]. See [`PreTrainedTokenizer.__call__`] and
+ [`PreTrainedTokenizer.encode`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`Numpy array` or `tf.Tensor` of shape `({0})`, *optional*):
@@ -731,14 +730,16 @@ ALBERT_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`Numpy array` or `tf.Tensor` of shape `({0})`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`Numpy array` or `tf.Tensor` of shape `({0})`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
head_mask (`Numpy array` or `tf.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
@@ -748,9 +749,9 @@ ALBERT_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
inputs_embeds (`tf.Tensor` of shape `({0}, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the
@@ -760,8 +761,8 @@ ALBERT_INPUTS_DOCSTRING = r"""
more detail. This argument can be used only in eager mode, in graph mode the value in the config will be
used instead.
return_dict (`bool`, *optional*):
- Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This
- argument can be used in eager mode, in graph mode the value will always be set to True.
+ Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This argument can be used
+ in eager mode, in graph mode the value will always be set to True.
training (`bool`, *optional*, defaults to `False`):
Whether or not to use the model in training mode (some modules like dropout modules have different
behaviors between training and evaluation).
@@ -982,7 +983,7 @@ class TFAlbertSOPHead(tf.keras.layers.Layer):
return logits
-@add_start_docstrings("""Albert Model with a `language modeling` head on top. """, ALBERT_START_DOCSTRING)
+@add_start_docstrings("""Albert Model with a `language modeling` head on top.""", ALBERT_START_DOCSTRING)
class TFAlbertForMaskedLM(TFAlbertPreTrainedModel, TFMaskedLanguageModelingLoss):
# names with a '.' represents the authorized unexpected/missing layers when a TF model is loaded from a PT model
_keys_to_ignore_on_load_unexpected = [r"pooler", r"predictions.decoder.weight"]
@@ -1020,8 +1021,9 @@ class TFAlbertForMaskedLM(TFAlbertPreTrainedModel, TFMaskedLanguageModelingLoss)
) -> Union[TFMaskedLMOutput, Tuple[tf.Tensor]]:
r"""
labels (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
"""
inputs = input_processing(
func=self.call,
@@ -1124,8 +1126,9 @@ class TFAlbertForSequenceClassification(TFAlbertPreTrainedModel, TFSequenceClass
) -> Union[TFSequenceClassifierOutput, Tuple[tf.Tensor]]:
r"""
labels (`tf.Tensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
inputs = input_processing(
func=self.call,
@@ -1334,12 +1337,12 @@ class TFAlbertForQuestionAnswering(TFAlbertPreTrainedModel, TFQuestionAnsweringL
r"""
start_positions (`tf.Tensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the start of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
end_positions (`tf.Tensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the end of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
"""
inputs = input_processing(
func=self.call,
@@ -1460,8 +1463,8 @@ class TFAlbertForMultipleChoice(TFAlbertPreTrainedModel, TFMultipleChoiceLoss):
) -> Union[TFMultipleChoiceModelOutput, Tuple[tf.Tensor]]:
r"""
labels (`tf.Tensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices]` where `num_choices` is the size of the second dimension of the input tensors. (See
- `input_ids` above)
+ Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices]`
+ where `num_choices` is the size of the second dimension of the input tensors. (See `input_ids` above)
"""
inputs = input_processing(
func=self.call,
diff --git a/src/transformers/models/albert/tokenization_albert.py b/src/transformers/models/albert/tokenization_albert.py
index 1d91860d0d..ba873afbab 100644
--- a/src/transformers/models/albert/tokenization_albert.py
+++ b/src/transformers/models/albert/tokenization_albert.py
@@ -60,8 +60,8 @@ class AlbertTokenizer(PreTrainedTokenizer):
"""
Construct an ALBERT tokenizer. Based on [SentencePiece](https://github.com/google/sentencepiece).
- This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods.
- Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
+ this superclass for more information regarding those methods.
Args:
vocab_file (`str`):
@@ -88,8 +88,8 @@ class AlbertTokenizer(PreTrainedTokenizer):
- When building a sequence using special tokens, this is not the token that is used for the end of
- sequence. The token used is the `sep_token`.
+ When building a sequence using special tokens, this is not the token that is used for the end of sequence.
+ The token used is the `sep_token`.
@@ -109,7 +109,9 @@ class AlbertTokenizer(PreTrainedTokenizer):
The token used for masking values. This is the token used when training this model with masked language
modeling. This is the token which the model will try to predict.
sp_model_kwargs (`dict`, *optional*):
- Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things, to set:
+ Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for
+ SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things,
+ to set:
- `enable_sampling`: Enable subword regularization.
- `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout.
@@ -324,8 +326,7 @@ class AlbertTokenizer(PreTrainedTokenizer):
Optional second list of IDs for sequence pairs.
Returns:
- `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given
- sequence(s).
+ `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
"""
sep = [self.sep_token_id]
cls = [self.cls_token_id]
diff --git a/src/transformers/models/albert/tokenization_albert_fast.py b/src/transformers/models/albert/tokenization_albert_fast.py
index 3c5adfbd21..8deb367851 100644
--- a/src/transformers/models/albert/tokenization_albert_fast.py
+++ b/src/transformers/models/albert/tokenization_albert_fast.py
@@ -72,9 +72,10 @@ SPIECE_UNDERLINE = "▁"
class AlbertTokenizerFast(PreTrainedTokenizerFast):
"""
- Construct a "fast" ALBERT tokenizer (backed by HuggingFace's *tokenizers* library). Based on [Unigram](https://huggingface.co/docs/tokenizers/python/latest/components.html?highlight=unigram#models). This tokenizer
- inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should
- refer to this superclass for more information regarding those methods
+ Construct a "fast" ALBERT tokenizer (backed by HuggingFace's *tokenizers* library). Based on
+ [Unigram](https://huggingface.co/docs/tokenizers/python/latest/components.html?highlight=unigram#models). This
+ tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should refer to
+ this superclass for more information regarding those methods
Args:
vocab_file (`str`):
@@ -213,8 +214,7 @@ class AlbertTokenizerFast(PreTrainedTokenizerFast):
Optional second list of IDs for sequence pairs.
Returns:
- `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given
- sequence(s).
+ `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
"""
sep = [self.sep_token_id]
cls = [self.cls_token_id]
diff --git a/src/transformers/models/auto/auto_factory.py b/src/transformers/models/auto/auto_factory.py
index b9afb0d363..c5914869bf 100644
--- a/src/transformers/models/auto/auto_factory.py
+++ b/src/transformers/models/auto/auto_factory.py
@@ -28,8 +28,8 @@ logger = logging.get_logger(__name__)
CLASS_DOCSTRING = """
This is a generic model class that will be instantiated as one of the model classes of the library when created
- with the [`~BaseAutoModelClass.from_pretrained`] class method or the
- [`~BaseAutoModelClass.from_config`] class method.
+ with the [`~BaseAutoModelClass.from_pretrained`] class method or the [`~BaseAutoModelClass.from_config`] class
+ method.
This class cannot be instantiated directly using `__init__()` (throws an error).
"""
@@ -39,8 +39,7 @@ FROM_CONFIG_DOCSTRING = """
Note:
Loading a model from its configuration file does **not** load the model weights. It only affects the
- model's configuration. Use [`~BaseAutoModelClass.from_pretrained`] to load the model
- weights.
+ model's configuration. Use [`~BaseAutoModelClass.from_pretrained`] to load the model weights.
Args:
config ([`PretrainedConfig`]):
@@ -62,8 +61,8 @@ FROM_PRETRAINED_TORCH_DOCSTRING = """
Instantiate one of the model classes of the library from a pretrained model.
The model class to instantiate is selected based on the `model_type` property of the config object (either
- passed as an argument or loaded from `pretrained_model_name_or_path` if possible), or when it's missing,
- by falling back to using pattern matching on `pretrained_model_name_or_path`:
+ passed as an argument or loaded from `pretrained_model_name_or_path` if possible), or when it's missing, by
+ falling back to using pattern matching on `pretrained_model_name_or_path`:
List options
@@ -75,14 +74,14 @@ FROM_PRETRAINED_TORCH_DOCSTRING = """
Can be either:
- A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co.
- Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under
- a user or organization name, like `dbmdz/bert-base-german-cased`.
+ Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under a
+ user or organization name, like `dbmdz/bert-base-german-cased`.
- A path to a *directory* containing model weights saved using
[`~PreTrainedModel.save_pretrained`], e.g., `./my_model_directory/`.
- A path or url to a *tensorflow index checkpoint file* (e.g, `./tf_model/model.ckpt.index`). In
- this case, `from_tf` should be set to `True` and a configuration object should be provided
- as `config` argument. This loading path is slower than converting the TensorFlow checkpoint in
- a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards.
+ this case, `from_tf` should be set to `True` and a configuration object should be provided as
+ `config` argument. This loading path is slower than converting the TensorFlow checkpoint in a
+ PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards.
model_args (additional positional arguments, *optional*):
Will be passed along to the underlying model `__init__()` method.
config ([`PretrainedConfig`], *optional*):
@@ -91,16 +90,15 @@ FROM_PRETRAINED_TORCH_DOCSTRING = """
- The model is a model provided by the library (loaded with the *model id* string of a pretrained
model).
- - The model was saved using [`~PreTrainedModel.save_pretrained`] and is reloaded
- by supplying the save directory.
+ - The model was saved using [`~PreTrainedModel.save_pretrained`] and is reloaded by supplying the
+ save directory.
- The model is loaded by supplying a local directory as `pretrained_model_name_or_path` and a
configuration JSON file named *config.json* is found in the directory.
state_dict (*Dict[str, torch.Tensor]*, *optional*):
A state dictionary to use instead of a state dictionary loaded from saved weights file.
This option can be used if you want to create a model from a pretrained configuration but load your own
- weights. In this case though, you should check if using
- [`~PreTrainedModel.save_pretrained`] and
+ weights. In this case though, you should check if using [`~PreTrainedModel.save_pretrained`] and
[`~PreTrainedModel.from_pretrained`] is not a simpler option.
cache_dir (`str` or `os.PathLike`, *optional*):
Path to a directory in which a downloaded pretrained model configuration should be cached if the
@@ -115,7 +113,8 @@ FROM_PRETRAINED_TORCH_DOCSTRING = """
Whether or not to delete incompletely received files. Will attempt to resume the download if such a
file exists.
proxies (`Dict[str, str]`, *optional*):
- A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request.
+ A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',
+ 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request.
output_loading_info(`bool`, *optional*, defaults to `False`):
Whether ot not to also return a dictionary containing missing keys, unexpected keys and error messages.
local_files_only(`bool`, *optional*, defaults to `False`):
@@ -126,8 +125,8 @@ FROM_PRETRAINED_TORCH_DOCSTRING = """
identifier allowed by git.
trust_remote_code (`bool`, *optional*, defaults to `False`):
Whether or not to allow for custom models defined on the Hub in their own modeling files. This option
- should only be set to `True` for repositories you trust and in which you have read the code, as it
- will execute code present on the Hub on your local machine.
+ should only be set to `True` for repositories you trust and in which you have read the code, as it will
+ execute code present on the Hub on your local machine.
kwargs (additional keyword arguments, *optional*):
Can be used to update the configuration object (after it being loaded) and initiate the model (e.g.,
`output_attentions=True`). Behaves differently depending on whether a `config` is provided or
@@ -137,10 +136,10 @@ FROM_PRETRAINED_TORCH_DOCSTRING = """
underlying model's `__init__` method (we assume all relevant updates to the configuration have
already been done)
- If a configuration is not provided, `kwargs` will be first passed to the configuration class
- initialization function ([`~PretrainedConfig.from_pretrained`]). Each key of
- `kwargs` that corresponds to a configuration attribute will be used to override said attribute
- with the supplied `kwargs` value. Remaining keys that do not correspond to any configuration
- attribute will be passed to the underlying model's `__init__` function.
+ initialization function ([`~PretrainedConfig.from_pretrained`]). Each key of `kwargs` that
+ corresponds to a configuration attribute will be used to override said attribute with the
+ supplied `kwargs` value. Remaining keys that do not correspond to any configuration attribute
+ will be passed to the underlying model's `__init__` function.
Examples:
@@ -165,8 +164,8 @@ FROM_PRETRAINED_TF_DOCSTRING = """
Instantiate one of the model classes of the library from a pretrained model.
The model class to instantiate is selected based on the `model_type` property of the config object (either
- passed as an argument or loaded from `pretrained_model_name_or_path` if possible), or when it's missing,
- by falling back to using pattern matching on `pretrained_model_name_or_path`:
+ passed as an argument or loaded from `pretrained_model_name_or_path` if possible), or when it's missing, by
+ falling back to using pattern matching on `pretrained_model_name_or_path`:
List options
@@ -175,15 +174,14 @@ FROM_PRETRAINED_TF_DOCSTRING = """
Can be either:
- A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co.
- Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under
- a user or organization name, like `dbmdz/bert-base-german-cased`.
+ Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under a
+ user or organization name, like `dbmdz/bert-base-german-cased`.
- A path to a *directory* containing model weights saved using
[`~PreTrainedModel.save_pretrained`], e.g., `./my_model_directory/`.
- - A path or url to a *PyTorch state_dict save file* (e.g, `./pt_model/pytorch_model.bin`). In
- this case, `from_pt` should be set to `True` and a configuration object should be provided
- as `config` argument. This loading path is slower than converting the PyTorch model in a
- TensorFlow model using the provided conversion scripts and loading the TensorFlow model
- afterwards.
+ - A path or url to a *PyTorch state_dict save file* (e.g, `./pt_model/pytorch_model.bin`). In this
+ case, `from_pt` should be set to `True` and a configuration object should be provided as `config`
+ argument. This loading path is slower than converting the PyTorch model in a TensorFlow model
+ using the provided conversion scripts and loading the TensorFlow model afterwards.
model_args (additional positional arguments, *optional*):
Will be passed along to the underlying model `__init__()` method.
config ([`PretrainedConfig`], *optional*):
@@ -192,8 +190,8 @@ FROM_PRETRAINED_TF_DOCSTRING = """
- The model is a model provided by the library (loaded with the *model id* string of a pretrained
model).
- - The model was saved using [`~PreTrainedModel.save_pretrained`] and is reloaded
- by supplying the save directory.
+ - The model was saved using [`~PreTrainedModel.save_pretrained`] and is reloaded by supplying the
+ save directory.
- The model is loaded by supplying a local directory as `pretrained_model_name_or_path` and a
configuration JSON file named *config.json* is found in the directory.
cache_dir (`str` or `os.PathLike`, *optional*):
@@ -209,7 +207,8 @@ FROM_PRETRAINED_TF_DOCSTRING = """
Whether or not to delete incompletely received files. Will attempt to resume the download if such a
file exists.
proxies (`Dict[str, str]`, *optional*):
- A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request.
+ A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',
+ 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request.
output_loading_info(`bool`, *optional*, defaults to `False`):
Whether ot not to also return a dictionary containing missing keys, unexpected keys and error messages.
local_files_only(`bool`, *optional*, defaults to `False`):
@@ -220,8 +219,8 @@ FROM_PRETRAINED_TF_DOCSTRING = """
identifier allowed by git.
trust_remote_code (`bool`, *optional*, defaults to `False`):
Whether or not to allow for custom models defined on the Hub in their own modeling files. This option
- should only be set to `True` for repositories you trust and in which you have read the code, as it
- will execute code present on the Hub on your local machine.
+ should only be set to `True` for repositories you trust and in which you have read the code, as it will
+ execute code present on the Hub on your local machine.
kwargs (additional keyword arguments, *optional*):
Can be used to update the configuration object (after it being loaded) and initiate the model (e.g.,
`output_attentions=True`). Behaves differently depending on whether a `config` is provided or
@@ -231,10 +230,10 @@ FROM_PRETRAINED_TF_DOCSTRING = """
underlying model's `__init__` method (we assume all relevant updates to the configuration have
already been done)
- If a configuration is not provided, `kwargs` will be first passed to the configuration class
- initialization function ([`~PretrainedConfig.from_pretrained`]). Each key of
- `kwargs` that corresponds to a configuration attribute will be used to override said attribute
- with the supplied `kwargs` value. Remaining keys that do not correspond to any configuration
- attribute will be passed to the underlying model's `__init__` function.
+ initialization function ([`~PretrainedConfig.from_pretrained`]). Each key of `kwargs` that
+ corresponds to a configuration attribute will be used to override said attribute with the
+ supplied `kwargs` value. Remaining keys that do not correspond to any configuration attribute
+ will be passed to the underlying model's `__init__` function.
Examples:
@@ -259,8 +258,8 @@ FROM_PRETRAINED_FLAX_DOCSTRING = """
Instantiate one of the model classes of the library from a pretrained model.
The model class to instantiate is selected based on the `model_type` property of the config object (either
- passed as an argument or loaded from `pretrained_model_name_or_path` if possible), or when it's missing,
- by falling back to using pattern matching on `pretrained_model_name_or_path`:
+ passed as an argument or loaded from `pretrained_model_name_or_path` if possible), or when it's missing, by
+ falling back to using pattern matching on `pretrained_model_name_or_path`:
List options
@@ -269,15 +268,14 @@ FROM_PRETRAINED_FLAX_DOCSTRING = """
Can be either:
- A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co.
- Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under
- a user or organization name, like `dbmdz/bert-base-german-cased`.
+ Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under a
+ user or organization name, like `dbmdz/bert-base-german-cased`.
- A path to a *directory* containing model weights saved using
[`~PreTrainedModel.save_pretrained`], e.g., `./my_model_directory/`.
- - A path or url to a *PyTorch state_dict save file* (e.g, `./pt_model/pytorch_model.bin`). In
- this case, `from_pt` should be set to `True` and a configuration object should be provided
- as `config` argument. This loading path is slower than converting the PyTorch model in a
- TensorFlow model using the provided conversion scripts and loading the TensorFlow model
- afterwards.
+ - A path or url to a *PyTorch state_dict save file* (e.g, `./pt_model/pytorch_model.bin`). In this
+ case, `from_pt` should be set to `True` and a configuration object should be provided as `config`
+ argument. This loading path is slower than converting the PyTorch model in a TensorFlow model
+ using the provided conversion scripts and loading the TensorFlow model afterwards.
model_args (additional positional arguments, *optional*):
Will be passed along to the underlying model `__init__()` method.
config ([`PretrainedConfig`], *optional*):
@@ -286,8 +284,8 @@ FROM_PRETRAINED_FLAX_DOCSTRING = """
- The model is a model provided by the library (loaded with the *model id* string of a pretrained
model).
- - The model was saved using [`~PreTrainedModel.save_pretrained`] and is reloaded
- by supplying the save directory.
+ - The model was saved using [`~PreTrainedModel.save_pretrained`] and is reloaded by supplying the
+ save directory.
- The model is loaded by supplying a local directory as `pretrained_model_name_or_path` and a
configuration JSON file named *config.json* is found in the directory.
cache_dir (`str` or `os.PathLike`, *optional*):
@@ -303,7 +301,8 @@ FROM_PRETRAINED_FLAX_DOCSTRING = """
Whether or not to delete incompletely received files. Will attempt to resume the download if such a
file exists.
proxies (`Dict[str, str]`, *optional*):
- A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request.
+ A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',
+ 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request.
output_loading_info(`bool`, *optional*, defaults to `False`):
Whether ot not to also return a dictionary containing missing keys, unexpected keys and error messages.
local_files_only(`bool`, *optional*, defaults to `False`):
@@ -314,8 +313,8 @@ FROM_PRETRAINED_FLAX_DOCSTRING = """
identifier allowed by git.
trust_remote_code (`bool`, *optional*, defaults to `False`):
Whether or not to allow for custom models defined on the Hub in their own modeling files. This option
- should only be set to `True` for repositories you trust and in which you have read the code, as it
- will execute code present on the Hub on your local machine.
+ should only be set to `True` for repositories you trust and in which you have read the code, as it will
+ execute code present on the Hub on your local machine.
kwargs (additional keyword arguments, *optional*):
Can be used to update the configuration object (after it being loaded) and initiate the model (e.g.,
`output_attentions=True`). Behaves differently depending on whether a `config` is provided or
@@ -325,10 +324,10 @@ FROM_PRETRAINED_FLAX_DOCSTRING = """
underlying model's `__init__` method (we assume all relevant updates to the configuration have
already been done)
- If a configuration is not provided, `kwargs` will be first passed to the configuration class
- initialization function ([`~PretrainedConfig.from_pretrained`]). Each key of
- `kwargs` that corresponds to a configuration attribute will be used to override said attribute
- with the supplied `kwargs` value. Remaining keys that do not correspond to any configuration
- attribute will be passed to the underlying model's `__init__` function.
+ initialization function ([`~PretrainedConfig.from_pretrained`]). Each key of `kwargs` that
+ corresponds to a configuration attribute will be used to override said attribute with the
+ supplied `kwargs` value. Remaining keys that do not correspond to any configuration attribute
+ will be passed to the underlying model's `__init__` function.
Examples:
diff --git a/src/transformers/models/auto/configuration_auto.py b/src/transformers/models/auto/configuration_auto.py
index bfe9772036..219bd6fb93 100644
--- a/src/transformers/models/auto/configuration_auto.py
+++ b/src/transformers/models/auto/configuration_auto.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" Auto Config class. """
+""" Auto Config class."""
import importlib
import re
import warnings
@@ -501,9 +501,8 @@ class AutoConfig:
r"""
Instantiate one of the configuration classes of the library from a pretrained model configuration.
- The configuration class to instantiate is selected based on the `model_type` property of the config object
- that is loaded, or when it's missing, by falling back to using pattern matching on
- `pretrained_model_name_or_path`:
+ The configuration class to instantiate is selected based on the `model_type` property of the config object that
+ is loaded, or when it's missing, by falling back to using pattern matching on `pretrained_model_name_or_path`:
List options
@@ -515,8 +514,8 @@ class AutoConfig:
huggingface.co. Valid model ids can be located at the root-level, like `bert-base-uncased`, or
namespaced under a user or organization name, like `dbmdz/bert-base-german-cased`.
- A path to a *directory* containing a configuration file saved using the
- [`~PretrainedConfig.save_pretrained`] method, or the
- [`~PreTrainedModel.save_pretrained`] method, e.g., `./my_model_directory/`.
+ [`~PretrainedConfig.save_pretrained`] method, or the [`~PreTrainedModel.save_pretrained`] method,
+ e.g., `./my_model_directory/`.
- A path or url to a saved configuration JSON *file*, e.g.,
`./my_model_directory/configuration.json`.
cache_dir (`str` or `os.PathLike`, *optional*):
@@ -529,7 +528,8 @@ class AutoConfig:
Whether or not to delete incompletely received files. Will attempt to resume the download if such a
file exists.
proxies (`Dict[str, str]`, *optional*):
- A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request.
+ A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',
+ 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request.
revision(`str`, *optional*, defaults to `"main"`):
The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a
git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any
@@ -537,13 +537,13 @@ class AutoConfig:
return_unused_kwargs (`bool`, *optional*, defaults to `False`):
If `False`, then this function returns just the final configuration object.
- If `True`, then this functions returns a `Tuple(config, unused_kwargs)` where *unused_kwargs*
- is a dictionary consisting of the key/value pairs whose keys are not configuration attributes: i.e.,
- the part of `kwargs` which has not been used to update `config` and is otherwise ignored.
+ If `True`, then this functions returns a `Tuple(config, unused_kwargs)` where *unused_kwargs* is a
+ dictionary consisting of the key/value pairs whose keys are not configuration attributes: i.e., the
+ part of `kwargs` which has not been used to update `config` and is otherwise ignored.
trust_remote_code (`bool`, *optional*, defaults to `False`):
Whether or not to allow for custom models defined on the Hub in their own modeling files. This option
- should only be set to `True` for repositories you trust and in which you have read the code, as it
- will execute code present on the Hub on your local machine.
+ should only be set to `True` for repositories you trust and in which you have read the code, as it will
+ execute code present on the Hub on your local machine.
kwargs(additional keyword arguments, *optional*):
The values in kwargs of any keys which are configuration attributes will be used to override the loaded
values. Behavior concerning key/value pairs whose keys are *not* configuration attributes is controlled
diff --git a/src/transformers/models/auto/dynamic.py b/src/transformers/models/auto/dynamic.py
index 1185298d85..20b968ed6a 100644
--- a/src/transformers/models/auto/dynamic.py
+++ b/src/transformers/models/auto/dynamic.py
@@ -122,8 +122,8 @@ def get_class_from_dynamic_module(
- Calling this function will execute the code in the module file found locally or downloaded from the Hub. It
- should therefore only be called on trusted repos.
+ Calling this function will execute the code in the module file found locally or downloaded from the Hub. It should
+ therefore only be called on trusted repos.
@@ -132,8 +132,8 @@ def get_class_from_dynamic_module(
This can be either:
- a string, the *model id* of a pretrained model configuration hosted inside a model repo on
- huggingface.co. Valid model ids can be located at the root-level, like `bert-base-uncased`, or
- namespaced under a user or organization name, like `dbmdz/bert-base-german-cased`.
+ huggingface.co. Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced
+ under a user or organization name, like `dbmdz/bert-base-german-cased`.
- a path to a *directory* containing a configuration file saved using the
[`~PreTrainedTokenizer.save_pretrained`] method, e.g., `./my_model_directory/`.
@@ -150,10 +150,11 @@ def get_class_from_dynamic_module(
resume_download (`bool`, *optional*, defaults to `False`):
Whether or not to delete incompletely received file. Attempts to resume the download if such a file exists.
proxies (`Dict[str, str]`, *optional*):
- A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request.
+ A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',
+ 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request.
use_auth_token (`str` or *bool*, *optional*):
- The token to use as HTTP bearer authorization for remote files. If `True`, will use the token
- generated when running `transformers-cli login` (stored in `~/.huggingface`).
+ The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated
+ when running `transformers-cli login` (stored in `~/.huggingface`).
revision(`str`, *optional*, defaults to `"main"`):
The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a
git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any
diff --git a/src/transformers/models/auto/feature_extraction_auto.py b/src/transformers/models/auto/feature_extraction_auto.py
index d5b2213ec4..8d1f8cf380 100644
--- a/src/transformers/models/auto/feature_extraction_auto.py
+++ b/src/transformers/models/auto/feature_extraction_auto.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" AutoFeatureExtractor class. """
+""" AutoFeatureExtractor class."""
import importlib
import os
from collections import OrderedDict
@@ -82,9 +82,9 @@ class AutoFeatureExtractor:
r"""
Instantiate one of the feature extractor classes of the library from a pretrained model vocabulary.
- The feature extractor class to instantiate is selected based on the `model_type` property of the config
- object (either passed as an argument or loaded from `pretrained_model_name_or_path` if possible), or when
- it's missing, by falling back to using pattern matching on `pretrained_model_name_or_path`:
+ The feature extractor class to instantiate is selected based on the `model_type` property of the config object
+ (either passed as an argument or loaded from `pretrained_model_name_or_path` if possible), or when it's
+ missing, by falling back to using pattern matching on `pretrained_model_name_or_path`:
List options
@@ -110,19 +110,20 @@ class AutoFeatureExtractor:
Whether or not to delete incompletely received file. Attempts to resume the download if such a file
exists.
proxies (`Dict[str, str]`, *optional*):
- A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request.
+ A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',
+ 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request.
use_auth_token (`str` or *bool*, *optional*):
- The token to use as HTTP bearer authorization for remote files. If `True`, will use the token
- generated when running `transformers-cli login` (stored in `~/.huggingface`).
+ The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated
+ when running `transformers-cli login` (stored in `~/.huggingface`).
revision(`str`, *optional*, defaults to `"main"`):
The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a
git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any
identifier allowed by git.
return_unused_kwargs (`bool`, *optional*, defaults to `False`):
- If `False`, then this function returns just the final feature extractor object. If `True`,
- then this functions returns a `Tuple(feature_extractor, unused_kwargs)` where *unused_kwargs* is a
- dictionary consisting of the key/value pairs whose keys are not feature extractor attributes: i.e., the
- part of `kwargs` which has not been used to update `feature_extractor` and is otherwise ignored.
+ If `False`, then this function returns just the final feature extractor object. If `True`, then this
+ functions returns a `Tuple(feature_extractor, unused_kwargs)` where *unused_kwargs* is a dictionary
+ consisting of the key/value pairs whose keys are not feature extractor attributes: i.e., the part of
+ `kwargs` which has not been used to update `feature_extractor` and is otherwise ignored.
kwargs (`Dict[str, Any]`, *optional*):
The values in kwargs of any keys which are feature extractor attributes will be used to override the
loaded values. Behavior concerning key/value pairs whose keys are *not* feature extractor attributes is
diff --git a/src/transformers/models/auto/modeling_auto.py b/src/transformers/models/auto/modeling_auto.py
index 73d5089ed7..20da3248c9 100644
--- a/src/transformers/models/auto/modeling_auto.py
+++ b/src/transformers/models/auto/modeling_auto.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" Auto Model class. """
+""" Auto Model class."""
import warnings
from collections import OrderedDict
diff --git a/src/transformers/models/auto/modeling_flax_auto.py b/src/transformers/models/auto/modeling_flax_auto.py
index de17b2a8ee..0191648f0e 100644
--- a/src/transformers/models/auto/modeling_flax_auto.py
+++ b/src/transformers/models/auto/modeling_flax_auto.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" Auto Model class. """
+""" Auto Model class."""
from collections import OrderedDict
diff --git a/src/transformers/models/auto/modeling_tf_auto.py b/src/transformers/models/auto/modeling_tf_auto.py
index 12bcd2e296..0ebad9ded5 100644
--- a/src/transformers/models/auto/modeling_tf_auto.py
+++ b/src/transformers/models/auto/modeling_tf_auto.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" Auto Model class. """
+""" Auto Model class."""
import warnings
diff --git a/src/transformers/models/auto/processing_auto.py b/src/transformers/models/auto/processing_auto.py
index f9f4868369..0a49dc24f5 100644
--- a/src/transformers/models/auto/processing_auto.py
+++ b/src/transformers/models/auto/processing_auto.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" AutoProcessor class. """
+""" AutoProcessor class."""
import importlib
from collections import OrderedDict
@@ -79,8 +79,8 @@ class AutoProcessor:
r"""
Instantiate one of the processor classes of the library from a pretrained model vocabulary.
- The processor class to instantiate is selected based on the `model_type` property of the config object
- (either passed as an argument or loaded from `pretrained_model_name_or_path` if possible):
+ The processor class to instantiate is selected based on the `model_type` property of the config object (either
+ passed as an argument or loaded from `pretrained_model_name_or_path` if possible):
List options
@@ -103,19 +103,20 @@ class AutoProcessor:
Whether or not to delete incompletely received file. Attempts to resume the download if such a file
exists.
proxies (`Dict[str, str]`, *optional*):
- A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request.
+ A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',
+ 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request.
use_auth_token (`str` or *bool*, *optional*):
- The token to use as HTTP bearer authorization for remote files. If `True`, will use the token
- generated when running `transformers-cli login` (stored in `~/.huggingface`).
+ The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated
+ when running `transformers-cli login` (stored in `~/.huggingface`).
revision (`str`, *optional*, defaults to `"main"`):
The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a
git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any
identifier allowed by git.
return_unused_kwargs (`bool`, *optional*, defaults to `False`):
- If `False`, then this function returns just the final feature extractor object. If `True`,
- then this functions returns a `Tuple(feature_extractor, unused_kwargs)` where *unused_kwargs* is a
- dictionary consisting of the key/value pairs whose keys are not feature extractor attributes: i.e., the
- part of `kwargs` which has not been used to update `feature_extractor` and is otherwise ignored.
+ If `False`, then this function returns just the final feature extractor object. If `True`, then this
+ functions returns a `Tuple(feature_extractor, unused_kwargs)` where *unused_kwargs* is a dictionary
+ consisting of the key/value pairs whose keys are not feature extractor attributes: i.e., the part of
+ `kwargs` which has not been used to update `feature_extractor` and is otherwise ignored.
kwargs (`Dict[str, Any]`, *optional*):
The values in kwargs of any keys which are feature extractor attributes will be used to override the
loaded values. Behavior concerning key/value pairs whose keys are *not* feature extractor attributes is
diff --git a/src/transformers/models/auto/tokenization_auto.py b/src/transformers/models/auto/tokenization_auto.py
index 68ecf76c7b..d73d56b27f 100644
--- a/src/transformers/models/auto/tokenization_auto.py
+++ b/src/transformers/models/auto/tokenization_auto.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" Auto Tokenizer class. """
+""" Auto Tokenizer class."""
import importlib
import json
@@ -279,8 +279,8 @@ def get_tokenizer_config(
This can be either:
- a string, the *model id* of a pretrained model configuration hosted inside a model repo on
- huggingface.co. Valid model ids can be located at the root-level, like `bert-base-uncased`, or
- namespaced under a user or organization name, like `dbmdz/bert-base-german-cased`.
+ huggingface.co. Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced
+ under a user or organization name, like `dbmdz/bert-base-german-cased`.
- a path to a *directory* containing a configuration file saved using the
[`~PreTrainedTokenizer.save_pretrained`] method, e.g., `./my_model_directory/`.
@@ -293,10 +293,11 @@ def get_tokenizer_config(
resume_download (`bool`, *optional*, defaults to `False`):
Whether or not to delete incompletely received file. Attempts to resume the download if such a file exists.
proxies (`Dict[str, str]`, *optional*):
- A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request.
+ A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',
+ 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request.
use_auth_token (`str` or *bool*, *optional*):
- The token to use as HTTP bearer authorization for remote files. If `True`, will use the token
- generated when running `transformers-cli login` (stored in `~/.huggingface`).
+ The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated
+ when running `transformers-cli login` (stored in `~/.huggingface`).
revision(`str`, *optional*, defaults to `"main"`):
The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a
git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any
@@ -390,9 +391,9 @@ class AutoTokenizer:
r"""
Instantiate one of the tokenizer classes of the library from a pretrained model vocabulary.
- The tokenizer class to instantiate is selected based on the `model_type` property of the config object
- (either passed as an argument or loaded from `pretrained_model_name_or_path` if possible), or when it's
- missing, by falling back to using pattern matching on `pretrained_model_name_or_path`:
+ The tokenizer class to instantiate is selected based on the `model_type` property of the config object (either
+ passed as an argument or loaded from `pretrained_model_name_or_path` if possible), or when it's missing, by
+ falling back to using pattern matching on `pretrained_model_name_or_path`:
List options
@@ -401,11 +402,10 @@ class AutoTokenizer:
Can be either:
- A string, the *model id* of a predefined tokenizer hosted inside a model repo on huggingface.co.
- Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under
- a user or organization name, like `dbmdz/bert-base-german-cased`.
+ Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under a
+ user or organization name, like `dbmdz/bert-base-german-cased`.
- A path to a *directory* containing vocabulary files required by the tokenizer, for instance saved
- using the [`~PreTrainedTokenizer.save_pretrained`] method, e.g.,
- `./my_model_directory/`.
+ using the [`~PreTrainedTokenizer.save_pretrained`] method, e.g., `./my_model_directory/`.
- A path or url to a single saved vocabulary file if and only if the tokenizer only requires a
single vocabulary file (like Bert or XLNet), e.g.: `./my_model_directory/vocab.txt`. (Not
applicable to all derived classes)
@@ -423,7 +423,8 @@ class AutoTokenizer:
Whether or not to delete incompletely received files. Will attempt to resume the download if such a
file exists.
proxies (`Dict[str, str]`, *optional*):
- A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request.
+ A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',
+ 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request.
revision(`str`, *optional*, defaults to `"main"`):
The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a
git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any
@@ -437,12 +438,12 @@ class AutoTokenizer:
Tokenizer type to be loaded.
trust_remote_code (`bool`, *optional*, defaults to `False`):
Whether or not to allow for custom models defined on the Hub in their own modeling files. This option
- should only be set to `True` for repositories you trust and in which you have read the code, as it
- will execute code present on the Hub on your local machine.
+ should only be set to `True` for repositories you trust and in which you have read the code, as it will
+ execute code present on the Hub on your local machine.
kwargs (additional keyword arguments, *optional*):
Will be passed to the Tokenizer `__init__()` method. Can be used to set special tokens like
- `bos_token`, `eos_token`, `unk_token`, `sep_token`, `pad_token`, `cls_token`,
- `mask_token`, `additional_special_tokens`. See parameters in the `__init__()` for more details.
+ `bos_token`, `eos_token`, `unk_token`, `sep_token`, `pad_token`, `cls_token`, `mask_token`,
+ `additional_special_tokens`. See parameters in the `__init__()` for more details.
Examples:
diff --git a/src/transformers/models/bart/configuration_bart.py b/src/transformers/models/bart/configuration_bart.py
index 05854e9bb7..81d9c12d81 100644
--- a/src/transformers/models/bart/configuration_bart.py
+++ b/src/transformers/models/bart/configuration_bart.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" BART model configuration """
+""" BART model configuration"""
import warnings
from collections import OrderedDict
from typing import Any, Mapping, Optional
@@ -35,19 +35,19 @@ BART_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class BartConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`BartModel`]. It is used to
- instantiate a BART model according to the specified arguments, defining the model architecture. Instantiating a
- configuration with the defaults will yield a similar configuration to that of the BART [facebook/bart-large](https://huggingface.co/facebook/bart-large) architecture.
+ This is the configuration class to store the configuration of a [`BartModel`]. It is used to instantiate a BART
+ model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
+ defaults will yield a similar configuration to that of the BART
+ [facebook/bart-large](https://huggingface.co/facebook/bart-large) architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
vocab_size (`int`, *optional*, defaults to 50265):
Vocabulary size of the BART model. Defines the number of different tokens that can be represented by the
- `inputs_ids` passed when calling [`BartModel`] or
- [`TFBartModel`].
+ `inputs_ids` passed when calling [`BartModel`] or [`TFBartModel`].
d_model (`int`, *optional*, defaults to 1024):
Dimensionality of the layers and the pooler layer.
encoder_layers (`int`, *optional*, defaults to 12):
@@ -63,8 +63,8 @@ class BartConfig(PretrainedConfig):
encoder_ffn_dim (`int`, *optional*, defaults to 4096):
Dimensionality of the "intermediate" (often named feed-forward) layer in decoder.
activation_function (`str` or `function`, *optional*, defaults to `"gelu"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"silu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"silu"` and `"gelu_new"` are supported.
dropout (`float`, *optional*, defaults to 0.1):
The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
attention_dropout (`float`, *optional*, defaults to 0.0):
@@ -79,11 +79,11 @@ class BartConfig(PretrainedConfig):
init_std (`float`, *optional*, defaults to 0.02):
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
encoder_layerdrop: (`float`, *optional*, defaults to 0.0):
- The LayerDrop probability for the encoder. See the [LayerDrop paper](see
- https://arxiv.org/abs/1909.11556) for more details.
+ The LayerDrop probability for the encoder. See the [LayerDrop paper](see https://arxiv.org/abs/1909.11556)
+ for more details.
decoder_layerdrop: (`float`, *optional*, defaults to 0.0):
- The LayerDrop probability for the decoder. See the [LayerDrop paper](see
- https://arxiv.org/abs/1909.11556) for more details.
+ The LayerDrop probability for the decoder. See the [LayerDrop paper](see https://arxiv.org/abs/1909.11556)
+ for more details.
scale_embedding (`bool`, *optional*, defaults to `False`):
Scale embeddings by diving by sqrt(d_model).
use_cache (`bool`, *optional*, defaults to `True`):
diff --git a/src/transformers/models/bart/modeling_bart.py b/src/transformers/models/bart/modeling_bart.py
index 534a19c3fd..5862b9595b 100755
--- a/src/transformers/models/bart/modeling_bart.py
+++ b/src/transformers/models/bart/modeling_bart.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" PyTorch BART model. """
+""" PyTorch BART model."""
import copy
import math
import random
@@ -380,7 +380,8 @@ class BartDecoderLayer(nn.Module):
hidden_states (`torch.FloatTensor`): input to the layer of shape *(batch, seq_len, embed_dim)*
attention_mask (`torch.FloatTensor`): attention mask of size
*(batch, 1, tgt_len, src_len)* where padding elements are indicated by very large negative values.
- encoder_hidden_states (`torch.FloatTensor`): cross attention input to the layer of shape *(batch, seq_len, embed_dim)*
+ encoder_hidden_states (`torch.FloatTensor`):
+ cross attention input to the layer of shape *(batch, seq_len, embed_dim)*
encoder_attention_mask (`torch.FloatTensor`): encoder attention mask of size
*(batch, 1, tgt_len, src_len)* where padding elements are indicated by very large negative values.
layer_head_mask (`torch.FloatTensor`): mask for attention heads in a given layer of size
@@ -517,13 +518,13 @@ class PretrainedBartModel(BartPretrainedModel):
BART_START_DOCSTRING = r"""
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
Parameters:
config ([`BartConfig`]):
@@ -537,29 +538,27 @@ BART_GENERATION_EXAMPLE = r"""
>>> from transformers import BartTokenizer, BartForConditionalGeneration, BartConfig
- >>> model = BartForConditionalGeneration.from_pretrained('facebook/bart-large-cnn')
- >>> tokenizer = BartTokenizer.from_pretrained('facebook/bart-large-cnn')
+ >>> model = BartForConditionalGeneration.from_pretrained('facebook/bart-large-cnn') >>> tokenizer =
+ BartTokenizer.from_pretrained('facebook/bart-large-cnn')
- >>> ARTICLE_TO_SUMMARIZE = "My friends are cool but they eat too many carbs."
- >>> inputs = tokenizer([ARTICLE_TO_SUMMARIZE], max_length=1024, return_tensors='pt')
+ >>> ARTICLE_TO_SUMMARIZE = "My friends are cool but they eat too many carbs." >>> inputs =
+ tokenizer([ARTICLE_TO_SUMMARIZE], max_length=1024, return_tensors='pt')
- >>> # Generate Summary
- >>> summary_ids = model.generate(inputs['input_ids'], num_beams=4, max_length=5, early_stopping=True)
- >>> print([tokenizer.decode(g, skip_special_tokens=True, clean_up_tokenization_spaces=False) for g in summary_ids])
+ >>> # Generate Summary >>> summary_ids = model.generate(inputs['input_ids'], num_beams=4, max_length=5,
+ early_stopping=True) >>> print([tokenizer.decode(g, skip_special_tokens=True,
+ clean_up_tokenization_spaces=False) for g in summary_ids])
Mask filling example::
- >>> from transformers import BartTokenizer, BartForConditionalGeneration
- >>> tokenizer = BartTokenizer.from_pretrained('facebook/bart-large')
- >>> TXT = "My friends are but they eat too many carbs."
+ >>> from transformers import BartTokenizer, BartForConditionalGeneration >>> tokenizer =
+ BartTokenizer.from_pretrained('facebook/bart-large') >>> TXT = "My friends are but they eat too many
+ carbs."
- >>> model = BartForConditionalGeneration.from_pretrained('facebook/bart-large')
- >>> input_ids = tokenizer([TXT], return_tensors='pt')['input_ids']
- >>> logits = model(input_ids).logits
+ >>> model = BartForConditionalGeneration.from_pretrained('facebook/bart-large') >>> input_ids =
+ tokenizer([TXT], return_tensors='pt')['input_ids'] >>> logits = model(input_ids).logits
- >>> masked_index = (input_ids[0] == tokenizer.mask_token_id).nonzero().item()
- >>> probs = logits[0, masked_index].softmax(dim=0)
- >>> values, predictions = probs.topk(5)
+ >>> masked_index = (input_ids[0] == tokenizer.mask_token_id).nonzero().item() >>> probs = logits[0,
+ masked_index].softmax(dim=0) >>> values, predictions = probs.topk(5)
>>> tokenizer.decode(predictions).split()
"""
@@ -570,9 +569,8 @@ BART_INPUTS_DOCSTRING = r"""
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
it.
- Indices can be obtained using [`BartTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`BartTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -585,26 +583,24 @@ BART_INPUTS_DOCSTRING = r"""
decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
Indices of decoder input sequence tokens in the vocabulary.
- Indices can be obtained using [`BartTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`BartTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are decoder input IDs?](../glossary#decoder-input-ids)
- Bart uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If
- `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
- `past_key_values`).
+ Bart uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values`
+ is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`).
For translation and summarization training, `decoder_input_ids` should be provided. If no
- `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to
- the right for denoising pre-training following the paper.
+ `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right
+ for denoising pre-training following the paper.
decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
- Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will
- also be used by default.
+ Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
+ be used by default.
If you want to change padding behavior, you should read [`modeling_bart._prepare_decoder_inputs`] and
- modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more
- information on the default strategy.
+ modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more information
+ on the default strategy.
head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*):
Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`:
@@ -618,39 +614,42 @@ BART_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*):
- Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, 1]`:
+ Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0,
+ 1]`:
- 1 indicates the head is **not masked**,
- 0 indicates the head is **masked**.
encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*):
- Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*:
- `attentions`) `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`,
- *optional*) is a sequence of hidden-states at the output of the last layer of the encoder. Used in the
- cross-attention of the decoder.
+ Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`)
+ `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of
+ hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder.
past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors
- of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
- shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
+ Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
+ `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape
+ `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all ``decoder_input_ids``` of shape `(batch_size, sequence_length)`. inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ ``decoder_input_ids``` of shape `(batch_size, sequence_length)`. inputs_embeds (`torch.FloatTensor` of
+ shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids`
+ you can choose to directly pass an embedded representation. This is useful if you want more control over
+ how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup
+ matrix.
decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*):
Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded
- representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds`
- have to be input (see `past_key_values`). This is useful if you want more control over how to convert
+ representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be
+ input (see `past_key_values`). This is useful if you want more control over how to convert
`decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix.
- If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds`
- takes the value of `inputs_embeds`.
+ If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds` takes the value
+ of `inputs_embeds`.
use_cache (`bool`, *optional*):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`).
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`).
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -721,9 +720,8 @@ class BartEncoder(BartPretrainedModel):
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
provide it.
- Indices can be obtained using [`BartTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`]
- for details.
+ Indices can be obtained using [`BartTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -740,9 +738,9 @@ class BartEncoder(BartPretrainedModel):
- 0 indicates the head is **masked**.
inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded
- representation. This is useful if you want more control over how to convert `input_ids` indices
- into associated vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
+ This is useful if you want more control over how to convert `input_ids` indices into associated vectors
+ than the model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under
returned tensors for more detail.
@@ -916,9 +914,8 @@ class BartDecoder(BartPretrainedModel):
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
provide it.
- Indices can be obtained using [`BartTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`]
- for details.
+ Indices can be obtained using [`BartTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -953,19 +950,20 @@ class BartDecoder(BartPretrainedModel):
- 0 indicates the head is **masked**.
past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2
- tensors of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional
- tensors of shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
+ Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of
+ shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
+ shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
Contains pre-computed hidden-states (key and values in the self-attention blocks and in the
- cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential
- decoding.
+ cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
- If `past_key_values` are used, the user can optionally input only the last
- `decoder_input_ids` (those that don't have their past key value states given to this model) of
- shape `(batch_size, 1)` instead of all ``decoder_input_ids``` of shape `(batch_size,
- sequence_length)`. inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices
- into associated vectors than the model's internal embedding lookup matrix.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those
+ that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of
+ all ``decoder_input_ids``` of shape `(batch_size, sequence_length)`. inputs_embeds (`torch.FloatTensor`
+ of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing
+ `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more
+ control over how to convert `input_ids` indices into associated vectors than the model's internal
+ embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under
returned tensors for more detail.
@@ -1304,7 +1302,8 @@ class BartForConditionalGeneration(BartPretrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
+ Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
+ config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
(masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
Returns:
@@ -1446,7 +1445,8 @@ class BartForSequenceClassification(BartPretrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
if labels is not None:
@@ -1712,9 +1712,8 @@ class BartForCausalLM(BartPretrainedModel):
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
provide it.
- Indices can be obtained using [`BartTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`]
- for details.
+ Indices can be obtained using [`BartTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -1743,25 +1742,24 @@ class BartForCausalLM(BartPretrainedModel):
- 0 indicates the head is **masked**.
past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2
- tensors of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional
- tensors of shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. The two
- additional tensors are only required when the model is used as a decoder in a Sequence to Sequence
- model.
+ Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of
+ shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
+ shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. The two additional
+ tensors are only required when the model is used as a decoder in a Sequence to Sequence model.
Contains pre-computed hidden-states (key and values in the self-attention blocks and in the
- cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential
- decoding.
+ cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those
+ that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of
+ all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are
- ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
+ Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
+ config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
+ (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
use_cache (`bool`, *optional*):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`).
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
+ (see `past_key_values`).
- 1 for tokens that are **not masked**,
- 0 for tokens that are **masked**.
diff --git a/src/transformers/models/bart/modeling_flax_bart.py b/src/transformers/models/bart/modeling_flax_bart.py
index 67a5872e65..d6cdcb7a01 100644
--- a/src/transformers/models/bart/modeling_flax_bart.py
+++ b/src/transformers/models/bart/modeling_flax_bart.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" Flax Bart model. """
+""" Flax Bart model."""
import math
import random
@@ -59,12 +59,13 @@ _TOKENIZER_FOR_DOC = "BartTokenizer"
BART_START_DOCSTRING = r"""
- This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading or saving, resizing the input
- embeddings, pruning heads etc.)
+ This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a Flax Linen [flax.nn.Module](https://flax.readthedocs.io/en/latest/_autosummary/flax.nn.module.html) subclass. Use it as a regular Flax
- Module and refer to the Flax documentation for all matter related to general usage and behavior.
+ This model is also a Flax Linen
+ [flax.nn.Module](https://flax.readthedocs.io/en/latest/_autosummary/flax.nn.module.html) subclass. Use it as a
+ regular Flax Module and refer to the Flax documentation for all matter related to general usage and behavior.
Finally, this model supports inherent JAX features such as:
@@ -76,11 +77,10 @@ BART_START_DOCSTRING = r"""
Parameters:
config ([`BartConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the
- model weights.
+ configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the model weights.
dtype (`jax.numpy.dtype`, *optional*, defaults to `jax.numpy.float32`):
- The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on
- GPUs) and `jax.numpy.bfloat16` (on TPUs).
+ The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on GPUs) and
+ `jax.numpy.bfloat16` (on TPUs).
This can be used to enable mixed-precision training or half-precision inference on GPUs or TPUs. If
specified all the computation will be performed with the given `dtype`.
@@ -88,8 +88,8 @@ BART_START_DOCSTRING = r"""
**Note that this only specifies the dtype of the computation and does not influence the dtype of model
parameters.**
- If you wish to change the dtype of the model parameters, see
- [`~FlaxPreTrainedModel.to_fp16`] and [`~FlaxPreTrainedModel.to_bf16`].
+ If you wish to change the dtype of the model parameters, see [`~FlaxPreTrainedModel.to_fp16`] and
+ [`~FlaxPreTrainedModel.to_bf16`].
"""
BART_INPUTS_DOCSTRING = r"""
@@ -98,9 +98,8 @@ BART_INPUTS_DOCSTRING = r"""
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
it.
- Indices can be obtained using [`BartTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`BartTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`jnp.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
@@ -113,22 +112,23 @@ BART_INPUTS_DOCSTRING = r"""
decoder_input_ids (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`, *optional*):
Indices of decoder input sequence tokens in the vocabulary.
- Indices can be obtained using [`BartTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`BartTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are decoder input IDs?](../glossary#decoder-input-ids)
For translation and summarization training, `decoder_input_ids` should be provided. If no
- `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to
- the right for denoising pre-training following the paper.
+ `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right
+ for denoising pre-training following the paper.
decoder_attention_mask (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`, *optional*):
- Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will
- also be used by default.
+ Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
+ be used by default.
- If you want to change padding behavior, you should modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy.
+ If you want to change padding behavior, you should modify to your needs. See diagram 1 in [the
+ paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy.
position_ids (`numpy.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
decoder_position_ids (`numpy.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the
range `[0, config.max_position_embeddings - 1]`.
@@ -149,9 +149,8 @@ BART_ENCODE_INPUTS_DOCSTRING = r"""
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
it.
- Indices can be obtained using [`BartTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`BartTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`jnp.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
@@ -162,7 +161,8 @@ BART_ENCODE_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
position_ids (`numpy.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -178,20 +178,18 @@ BART_DECODE_INPUTS_DOCSTRING = r"""
decoder_input_ids (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`):
Indices of decoder input sequence tokens in the vocabulary.
- Indices can be obtained using [`BartTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`BartTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are decoder input IDs?](../glossary#decoder-input-ids)
For translation and summarization training, `decoder_input_ids` should be provided. If no
- `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to
- the right for denoising pre-training following the paper.
+ `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right
+ for denoising pre-training following the paper.
encoder_outputs (`tuple(tuple(jnp.ndarray)`):
- Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*:
- `attentions`) `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`,
- *optional*) is a sequence of hidden-states at the output of the last layer of the encoder. Used in the
- cross-attention of the decoder.
+ Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`)
+ `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of
+ hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder.
encoder_attention_mask (`jnp.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
@@ -200,10 +198,11 @@ BART_DECODE_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
decoder_attention_mask (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`, *optional*):
- Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will
- also be used by default.
+ Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
+ be used by default.
- If you want to change padding behavior, you should modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy.
+ If you want to change padding behavior, you should modify to your needs. See diagram 1 in [the
+ paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy.
decoder_position_ids (`numpy.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the
range `[0, config.max_position_embeddings - 1]`.
@@ -968,9 +967,10 @@ class FlaxBartPreTrainedModel(FlaxPreTrainedModel):
maximum possible length for auto-regressive decoding. Defines the sequence length of the initialized
cache.
encoder_outputs (`Union[FlaxBaseModelOutput, tuple(tuple(jnp.ndarray)]`):
- `encoder_outputs` consists of (`last_hidden_state`, *optional*: `hidden_states`,
- *optional*: `attentions`). `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of hidden-states at the output of the last layer of the
- encoder. Used in the cross-attention of the decoder.
+ `encoder_outputs` consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*:
+ `attentions`). `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*)
+ is a sequence of hidden-states at the output of the last layer of the encoder. Used in the
+ cross-attention of the decoder.
"""
# init input variables to retrieve cache
decoder_input_ids = jnp.ones((batch_size, max_length), dtype="i4")
@@ -1510,29 +1510,26 @@ FLAX_BART_CONDITIONAL_GENERATION_DOCSTRING = """
>>> from transformers import BartTokenizer, FlaxBartForConditionalGeneration
- >>> model = FlaxBartForConditionalGeneration.from_pretrained('facebook/bart-large-cnn')
- >>> tokenizer = BartTokenizer.from_pretrained('facebook/bart-large-cnn')
+ >>> model = FlaxBartForConditionalGeneration.from_pretrained('facebook/bart-large-cnn') >>> tokenizer =
+ BartTokenizer.from_pretrained('facebook/bart-large-cnn')
- >>> ARTICLE_TO_SUMMARIZE = "My friends are cool but they eat too many carbs."
- >>> inputs = tokenizer([ARTICLE_TO_SUMMARIZE], max_length=1024, return_tensors='jax')
+ >>> ARTICLE_TO_SUMMARIZE = "My friends are cool but they eat too many carbs." >>> inputs =
+ tokenizer([ARTICLE_TO_SUMMARIZE], max_length=1024, return_tensors='jax')
- >>> # Generate Summary
- >>> summary_ids = model.generate(inputs['input_ids']).sequences
- >>> print(tokenizer.batch_decode(summary_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False))
+ >>> # Generate Summary >>> summary_ids = model.generate(inputs['input_ids']).sequences >>>
+ print(tokenizer.batch_decode(summary_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False))
Mask filling example::
- >>> from transformers import BartTokenizer, FlaxBartForConditionalGeneration
- >>> tokenizer = BartTokenizer.from_pretrained('facebook/bart-large')
- >>> TXT = "My friends are but they eat too many carbs."
+ >>> from transformers import BartTokenizer, FlaxBartForConditionalGeneration >>> tokenizer =
+ BartTokenizer.from_pretrained('facebook/bart-large') >>> TXT = "My friends are but they eat too many
+ carbs."
- >>> model = FlaxBartForConditionalGeneration.from_pretrained('facebook/bart-large')
- >>> input_ids = tokenizer([TXT], return_tensors='jax')['input_ids']
- >>> logits = model(input_ids).logits
+ >>> model = FlaxBartForConditionalGeneration.from_pretrained('facebook/bart-large') >>> input_ids =
+ tokenizer([TXT], return_tensors='jax')['input_ids'] >>> logits = model(input_ids).logits
- >>> masked_index = (input_ids[0] == tokenizer.mask_token_id).nonzero()[0].item()
- >>> probs = jax.nn.softmax(logits[0, masked_index], axis=0)
- >>> values, predictions = jax.lax.top_k(probs)
+ >>> masked_index = (input_ids[0] == tokenizer.mask_token_id).nonzero()[0].item() >>> probs =
+ jax.nn.softmax(logits[0, masked_index], axis=0) >>> values, predictions = jax.lax.top_k(probs)
>>> tokenizer.decode(predictions).split()
"""
diff --git a/src/transformers/models/bart/modeling_tf_bart.py b/src/transformers/models/bart/modeling_tf_bart.py
index abf5b4bb4e..04d32aecab 100644
--- a/src/transformers/models/bart/modeling_tf_bart.py
+++ b/src/transformers/models/bart/modeling_tf_bart.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" TF 2.0 Bart model. """
+""" TF 2.0 Bart model."""
import random
@@ -375,7 +375,8 @@ class TFBartDecoderLayer(tf.keras.layers.Layer):
hidden_states (`tf.Tensor`): input to the layer of shape *(seq_len, batch, embed_dim)*
attention_mask (`tf.Tensor`): attention mask of size
*(batch, 1, tgt_len, src_len)* where padding elements are indicated by very large negative values.
- encoder_hidden_states (`tf.Tensor`): cross attention input to the layer of shape *(seq_len, batch, embed_dim)*
+ encoder_hidden_states (`tf.Tensor`):
+ cross attention input to the layer of shape *(seq_len, batch, embed_dim)*
encoder_attention_mask (`tf.Tensor`): encoder attention mask of size
*(batch, 1, tgt_len, src_len)* where padding elements are indicated by very large negative values.
layer_head_mask (`tf.Tensor`): mask for attention heads in a given layer of size
@@ -472,13 +473,13 @@ class TFBartPretrainedModel(TFPreTrainedModel):
BART_START_DOCSTRING = r"""
- This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading or saving, resizing the input
- embeddings, pruning heads etc.)
+ This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use
- it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage
- and behavior.
+ This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it
+ as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and
+ behavior.
@@ -487,11 +488,11 @@ BART_START_DOCSTRING = r"""
- having all inputs as keyword arguments (like PyTorch models), or
- having all inputs as a list, tuple or dict in the first positional arguments.
- This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all
- the tensors in the first argument of the model call function: `model(inputs)`.
+ This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all the
+ tensors in the first argument of the model call function: `model(inputs)`.
- If you choose this second option, there are three possibilities you can use to gather all the input Tensors in
- the first positional argument :
+ If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the
+ first positional argument :
- a single Tensor with `input_ids` only and nothing else: `model(input_ids)`
- a list of varying length with one or several input Tensors IN THE ORDER given in the docstring:
@@ -504,8 +505,7 @@ BART_START_DOCSTRING = r"""
Args:
config ([`BartConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~TFPreTrainedModel.from_pretrained`] method to load the
- model weights.
+ configuration. Check out the [`~TFPreTrainedModel.from_pretrained`] method to load the model weights.
"""
@@ -514,27 +514,25 @@ BART_GENERATION_EXAMPLE = r"""
>>> from transformers import BartTokenizer, TFBartForConditionalGeneration, BartConfig
- >>> model = TFBartForConditionalGeneration.from_pretrained('facebook/bart-large')
- >>> tokenizer = BartTokenizer.from_pretrained('facebook/bart-large')
+ >>> model = TFBartForConditionalGeneration.from_pretrained('facebook/bart-large') >>> tokenizer =
+ BartTokenizer.from_pretrained('facebook/bart-large')
- >>> ARTICLE_TO_SUMMARIZE = "My friends are cool but they eat too many carbs."
- >>> inputs = tokenizer([ARTICLE_TO_SUMMARIZE], max_length=1024, return_tensors='tf')
+ >>> ARTICLE_TO_SUMMARIZE = "My friends are cool but they eat too many carbs." >>> inputs =
+ tokenizer([ARTICLE_TO_SUMMARIZE], max_length=1024, return_tensors='tf')
- >>> # Generate Summary
- >>> summary_ids = model.generate(inputs['input_ids'], num_beams=4, max_length=5, early_stopping=True)
- >>> print([tokenizer.decode(g, skip_special_tokens=True, clean_up_tokenization_spaces=False) for g in summary_ids])
+ >>> # Generate Summary >>> summary_ids = model.generate(inputs['input_ids'], num_beams=4, max_length=5,
+ early_stopping=True) >>> print([tokenizer.decode(g, skip_special_tokens=True,
+ clean_up_tokenization_spaces=False) for g in summary_ids])
Mask filling example::
- >>> from transformers import BartTokenizer, TFBartForConditionalGeneration
- >>> tokenizer = BartTokenizer.from_pretrained('facebook/bart-large')
- >>> TXT = "My friends are but they eat too many carbs."
+ >>> from transformers import BartTokenizer, TFBartForConditionalGeneration >>> tokenizer =
+ BartTokenizer.from_pretrained('facebook/bart-large') >>> TXT = "My friends are but they eat too many
+ carbs."
- >>> model = TFBartForConditionalGeneration.from_pretrained('facebook/bart-large')
- >>> input_ids = tokenizer([TXT], return_tensors='tf')['input_ids']
- >>> logits = model(input_ids).logits
- >>> probs = tf.nn.softmax(logits[0])
- >>> # probs[5] is associated with the mask token
+ >>> model = TFBartForConditionalGeneration.from_pretrained('facebook/bart-large') >>> input_ids =
+ tokenizer([TXT], return_tensors='tf')['input_ids'] >>> logits = model(input_ids).logits >>> probs =
+ tf.nn.softmax(logits[0]) >>> # probs[5] is associated with the mask token
"""
@@ -543,9 +541,8 @@ BART_INPUTS_DOCSTRING = r"""
input_ids (`tf.Tensor` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`BertTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`BertTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`tf.Tensor` of shape `({0})`, *optional*):
@@ -558,19 +555,17 @@ BART_INPUTS_DOCSTRING = r"""
decoder_input_ids (`tf.Tensor` of shape `(batch_size, target_sequence_length)`, *optional*):
Indices of decoder input sequence tokens in the vocabulary.
- Indices can be obtained using [`BartTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`BartTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are decoder input IDs?](../glossary#decoder-input-ids)
- Bart uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If
- `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
- `past_key_values`).
+ Bart uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values`
+ is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`).
For translation and summarization training, `decoder_input_ids` should be provided. If no
- `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to
- the right for denoising pre-training following the paper.
+ `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right
+ for denoising pre-training following the paper.
decoder_attention_mask (`tf.Tensor` of shape `(batch_size, target_sequence_length)`, *optional*):
will be made by default and ignore pad tokens. It is not recommended to set this for most use cases.
head_mask (`tf.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*):
@@ -596,12 +591,12 @@ BART_INPUTS_DOCSTRING = r"""
of shape `(batch_size, sequence_length, hidden_size)` is a sequence of
past_key_values (`Tuple[Tuple[tf.Tensor]]` of length `config.n_layers`)
contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ `decoder_input_ids` of shape `(batch_size, sequence_length)`.
use_cache (`bool`, *optional*, defaults to `True`):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`). Set to `False` during training, `True` during generation
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`). Set to `False` during training, `True` during generation
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the
@@ -611,8 +606,8 @@ BART_INPUTS_DOCSTRING = r"""
more detail. This argument can be used only in eager mode, in graph mode the value in the config will be
used instead.
return_dict (`bool`, *optional*):
- Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This
- argument can be used in eager mode, in graph mode the value will always be set to True.
+ Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This argument can be used
+ in eager mode, in graph mode the value will always be set to True.
training (`bool`, *optional*, defaults to `False`):
Whether or not to use the model in training mode (some modules like dropout modules have different
behaviors between training and evaluation).
@@ -672,9 +667,8 @@ class TFBartEncoder(tf.keras.layers.Layer):
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
provide it.
- Indices can be obtained using [`BartTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`]
- for details.
+ Indices can be obtained using [`BartTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -684,15 +678,16 @@ class TFBartEncoder(tf.keras.layers.Layer):
- 0 for tokens that are **masked**.
[What are attention masks?](../glossary#attention-mask)
- head_mask (`tf.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, `optional): Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`:
+ head_mask (`tf.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, `optional):
+ Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`:
- 1 indicates the head is **not masked**,
- 0 indicates the head is **masked**.
inputs_embeds (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded
- representation. This is useful if you want more control over how to convert `input_ids` indices
- into associated vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
+ This is useful if you want more control over how to convert `input_ids` indices into associated vectors
+ than the model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under
returned tensors for more detail.
@@ -839,9 +834,8 @@ class TFBartDecoder(tf.keras.layers.Layer):
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
provide it.
- Indices can be obtained using [`BartTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`]
- for details.
+ Indices can be obtained using [`BartTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -878,11 +872,13 @@ class TFBartDecoder(tf.keras.layers.Layer):
Contains precomputed key and value hidden-states of the attention blocks. Can be used to speed up
decoding.
- If `past_key_values` are used, the user can optionally input only the last
- `decoder_input_ids` (those that don't have their past key value states given to this model) of
- shape `(batch_size, 1)` instead of all ``decoder_input_ids``` of shape `(batch_size,
- sequence_length)`. inputs_embeds (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices
- into associated vectors than the model's internal embedding lookup matrix.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those
+ that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of
+ all ``decoder_input_ids``` of shape `(batch_size, sequence_length)`. inputs_embeds (`tf.Tensor` of
+ shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing
+ `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more
+ control over how to convert `input_ids` indices into associated vectors than the model's internal
+ embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under
returned tensors for more detail.
@@ -1347,7 +1343,8 @@ class TFBartForConditionalGeneration(TFBartPretrainedModel, TFCausalLanguageMode
):
r"""
labels (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
+ Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
+ config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
(masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
Returns:
diff --git a/src/transformers/models/bart/tokenization_bart.py b/src/transformers/models/bart/tokenization_bart.py
index ccd189ab1b..7edf2b7346 100644
--- a/src/transformers/models/bart/tokenization_bart.py
+++ b/src/transformers/models/bart/tokenization_bart.py
@@ -56,9 +56,8 @@ class BartTokenizer(RobertaTokenizer):
r"""
Construct a BART tokenizer.
- [`BartTokenizer`] is identical to [`RobertaTokenizer`]. Refer to superclass
- [`RobertaTokenizer`] for usage examples and documentation concerning the initialization
- parameters and other methods.
+ [`BartTokenizer`] is identical to [`RobertaTokenizer`]. Refer to superclass [`RobertaTokenizer`] for usage examples
+ and documentation concerning the initialization parameters and other methods.
"""
vocab_files_names = VOCAB_FILES_NAMES
pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
diff --git a/src/transformers/models/bart/tokenization_bart_fast.py b/src/transformers/models/bart/tokenization_bart_fast.py
index 33bda3efb1..d2d4fba5d5 100644
--- a/src/transformers/models/bart/tokenization_bart_fast.py
+++ b/src/transformers/models/bart/tokenization_bart_fast.py
@@ -65,9 +65,8 @@ class BartTokenizerFast(RobertaTokenizerFast):
r"""
Construct a "fast" BART tokenizer (backed by HuggingFace's *tokenizers* library).
- [`BartTokenizerFast`] is identical to [`RobertaTokenizerFast`]. Refer to
- superclass [`RobertaTokenizerFast`] for usage examples and documentation concerning the
- initialization parameters and other methods.
+ [`BartTokenizerFast`] is identical to [`RobertaTokenizerFast`]. Refer to superclass [`RobertaTokenizerFast`] for
+ usage examples and documentation concerning the initialization parameters and other methods.
"""
vocab_files_names = VOCAB_FILES_NAMES
pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
diff --git a/src/transformers/models/barthez/tokenization_barthez.py b/src/transformers/models/barthez/tokenization_barthez.py
index ea0ae5897b..f2a7d353c1 100644
--- a/src/transformers/models/barthez/tokenization_barthez.py
+++ b/src/transformers/models/barthez/tokenization_barthez.py
@@ -48,11 +48,11 @@ SPIECE_UNDERLINE = "▁"
class BarthezTokenizer(PreTrainedTokenizer):
"""
- Adapted from [`CamembertTokenizer`] and [`BartTokenizer`]. Construct a
- BARThez tokenizer. Based on [SentencePiece](https://github.com/google/sentencepiece).
+ Adapted from [`CamembertTokenizer`] and [`BartTokenizer`]. Construct a BARThez tokenizer. Based on
+ [SentencePiece](https://github.com/google/sentencepiece).
- This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods.
- Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
+ this superclass for more information regarding those methods.
Args:
vocab_file (`str`):
@@ -73,8 +73,8 @@ class BarthezTokenizer(PreTrainedTokenizer):
- When building a sequence using special tokens, this is not the token that is used for the end of
- sequence. The token used is the `sep_token`.
+ When building a sequence using special tokens, this is not the token that is used for the end of sequence.
+ The token used is the `sep_token`.
@@ -96,7 +96,9 @@ class BarthezTokenizer(PreTrainedTokenizer):
additional_special_tokens (`List[str]`, *optional*, defaults to `["NOTUSED", "NOTUSED"]`):
Additional special tokens used by the tokenizer.
sp_model_kwargs (`dict`, *optional*):
- Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things, to set:
+ Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for
+ SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things,
+ to set:
- `enable_sampling`: Enable subword regularization.
- `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout.
diff --git a/src/transformers/models/barthez/tokenization_barthez_fast.py b/src/transformers/models/barthez/tokenization_barthez_fast.py
index f896a331a0..e5e7b96a3a 100644
--- a/src/transformers/models/barthez/tokenization_barthez_fast.py
+++ b/src/transformers/models/barthez/tokenization_barthez_fast.py
@@ -58,11 +58,11 @@ SPIECE_UNDERLINE = "▁"
class BarthezTokenizerFast(PreTrainedTokenizerFast):
"""
- Adapted from [`CamembertTokenizer`] and [`BartTokenizer`]. Construct a "fast"
- BARThez tokenizer. Based on [SentencePiece](https://github.com/google/sentencepiece).
+ Adapted from [`CamembertTokenizer`] and [`BartTokenizer`]. Construct a "fast" BARThez tokenizer. Based on
+ [SentencePiece](https://github.com/google/sentencepiece).
- This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main
- methods. Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should
+ refer to this superclass for more information regarding those methods.
Args:
vocab_file (`str`):
@@ -83,8 +83,8 @@ class BarthezTokenizerFast(PreTrainedTokenizerFast):
- When building a sequence using special tokens, this is not the token that is used for the end of
- sequence. The token used is the `sep_token`.
+ When building a sequence using special tokens, this is not the token that is used for the end of sequence.
+ The token used is the `sep_token`.
diff --git a/src/transformers/models/bartpho/tokenization_bartpho.py b/src/transformers/models/bartpho/tokenization_bartpho.py
index c2e599371c..0bc17876af 100644
--- a/src/transformers/models/bartpho/tokenization_bartpho.py
+++ b/src/transformers/models/bartpho/tokenization_bartpho.py
@@ -47,8 +47,8 @@ class BartphoTokenizer(PreTrainedTokenizer):
"""
Adapted from [`XLMRobertaTokenizer`]. Based on [SentencePiece](https://github.com/google/sentencepiece).
- This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods.
- Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
+ this superclass for more information regarding those methods.
Args:
vocab_file (`str`):
@@ -72,8 +72,8 @@ class BartphoTokenizer(PreTrainedTokenizer):
- When building a sequence using special tokens, this is not the token that is used for the end of
- sequence. The token used is the `sep_token`.
+ When building a sequence using special tokens, this is not the token that is used for the end of sequence.
+ The token used is the `sep_token`.
@@ -95,7 +95,9 @@ class BartphoTokenizer(PreTrainedTokenizer):
additional_special_tokens (`List[str]`, *optional*, defaults to `["NOTUSED", "NOTUSED"]`):
Additional special tokens used by the tokenizer.
sp_model_kwargs (`dict`, *optional*):
- Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things, to set:
+ Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for
+ SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things,
+ to set:
- `enable_sampling`: Enable subword regularization.
- `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout.
diff --git a/src/transformers/models/beit/configuration_beit.py b/src/transformers/models/beit/configuration_beit.py
index 6634fc03b1..b668a2ed8c 100644
--- a/src/transformers/models/beit/configuration_beit.py
+++ b/src/transformers/models/beit/configuration_beit.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" BEiT model configuration """
+""" BEiT model configuration"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging
@@ -28,11 +28,10 @@ BEIT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class BeitConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`BeitModel`]. It is used to
- instantiate an BEiT model according to the specified arguments, defining the model architecture. Instantiating a
- configuration with the defaults will yield a similar configuration to that of the BEiT
- [microsoft/beit-base-patch16-224-in22k](https://huggingface.co/microsoft/beit-base-patch16-224-in22k)
- architecture.
+ This is the configuration class to store the configuration of a [`BeitModel`]. It is used to instantiate an BEiT
+ model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
+ defaults will yield a similar configuration to that of the BEiT
+ [microsoft/beit-base-patch16-224-in22k](https://huggingface.co/microsoft/beit-base-patch16-224-in22k) architecture.
Args:
vocab_size (`int`, *optional*, defaults to 8092):
@@ -47,8 +46,8 @@ class BeitConfig(PretrainedConfig):
intermediate_size (`int`, *optional*, defaults to 3072):
Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"selu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"selu"` and `"gelu_new"` are supported.
hidden_dropout_prob (`float`, *optional*, defaults to 0.1):
The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
attention_probs_dropout_prob (`float`, *optional*, defaults to 0.1):
diff --git a/src/transformers/models/beit/feature_extraction_beit.py b/src/transformers/models/beit/feature_extraction_beit.py
index 997f860115..2567bbee56 100644
--- a/src/transformers/models/beit/feature_extraction_beit.py
+++ b/src/transformers/models/beit/feature_extraction_beit.py
@@ -38,27 +38,25 @@ class BeitFeatureExtractor(FeatureExtractionMixin, ImageFeatureExtractionMixin):
r"""
Constructs a BEiT feature extractor.
- This feature extractor inherits from [`~feature_extraction_utils.FeatureExtractionMixin`] which
- contains most of the main methods. Users should refer to this superclass for more information regarding those
- methods.
+ This feature extractor inherits from [`~feature_extraction_utils.FeatureExtractionMixin`] which contains most of
+ the main methods. Users should refer to this superclass for more information regarding those methods.
Args:
do_resize (`bool`, *optional*, defaults to `True`):
Whether to resize the input to a certain `size`.
size (`int` or `Tuple(int)`, *optional*, defaults to 256):
Resize the input to the given size. If a tuple is provided, it should be (width, height). If only an
- integer is provided, then the input will be resized to (size, size). Only has an effect if `do_resize`
- is set to `True`.
+ integer is provided, then the input will be resized to (size, size). Only has an effect if `do_resize` is
+ set to `True`.
resample (`int`, *optional*, defaults to `PIL.Image.BICUBIC`):
An optional resampling filter. This can be one of `PIL.Image.NEAREST`, `PIL.Image.BOX`,
- `PIL.Image.BILINEAR`, `PIL.Image.HAMMING`, `PIL.Image.BICUBIC` or `PIL.Image.LANCZOS`.
- Only has an effect if `do_resize` is set to `True`.
+ `PIL.Image.BILINEAR`, `PIL.Image.HAMMING`, `PIL.Image.BICUBIC` or `PIL.Image.LANCZOS`. Only has an effect
+ if `do_resize` is set to `True`.
do_center_crop (`bool`, *optional*, defaults to `True`):
- Whether to crop the input at the center. If the input size is smaller than `crop_size` along any edge,
- the image is padded with 0's and then center cropped.
+ Whether to crop the input at the center. If the input size is smaller than `crop_size` along any edge, the
+ image is padded with 0's and then center cropped.
crop_size (`int`, *optional*, defaults to 224):
- Desired output size when applying center-cropping. Only has an effect if `do_center_crop` is set to
- `True`.
+ Desired output size when applying center-cropping. Only has an effect if `do_center_crop` is set to `True`.
do_normalize (`bool`, *optional*, defaults to `True`):
Whether or not to normalize the input with `image_mean` and `image_std`.
image_mean (`List[int]`, defaults to `[0.5, 0.5, 0.5]`):
diff --git a/src/transformers/models/beit/modeling_beit.py b/src/transformers/models/beit/modeling_beit.py
index 4dfa68af8f..19a2754148 100755
--- a/src/transformers/models/beit/modeling_beit.py
+++ b/src/transformers/models/beit/modeling_beit.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" PyTorch BEiT model. """
+""" PyTorch BEiT model."""
import collections.abc
@@ -56,12 +56,13 @@ class BeitModelOutputWithPooling(BaseModelOutputWithPooling):
*config.use_mean_pooling* is set to True. If set to False, then the final hidden state of the *[CLS]* token
will be returned.
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -547,15 +548,14 @@ class BeitPreTrainedModel(PreTrainedModel):
BEIT_START_DOCSTRING = r"""
- This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use
- it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
+ This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it
+ as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
behavior.
Parameters:
config ([`BeitConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
BEIT_INPUTS_DOCSTRING = r"""
@@ -737,8 +737,9 @@ class BeitForMaskedImageModeling(BeitPreTrainedModel):
Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the image classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
Returns:
@@ -824,8 +825,9 @@ class BeitForImageClassification(BeitPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the image classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
Returns:
@@ -1158,8 +1160,8 @@ class BeitForSemanticSegmentation(BeitPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size, height, width)`, *optional*):
- Ground truth semantic segmentation maps for computing the loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels > 1`, a classification loss is computed
- (Cross-Entropy).
+ Ground truth semantic segmentation maps for computing the loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels > 1`, a classification loss is computed (Cross-Entropy).
Returns:
diff --git a/src/transformers/models/beit/modeling_flax_beit.py b/src/transformers/models/beit/modeling_flax_beit.py
index b81f1afb41..19ae6fabe2 100644
--- a/src/transformers/models/beit/modeling_flax_beit.py
+++ b/src/transformers/models/beit/modeling_flax_beit.py
@@ -54,23 +54,24 @@ class FlaxBeitModelOutputWithPooling(FlaxBaseModelOutputWithPooling):
*config.use_mean_pooling* is set to True. If set to False, then the final hidden state of the *[CLS]* token
will be returned.
hidden_states (`tuple(jnp.ndarray)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each
- layer plus the initial embedding outputs.
+ Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer plus
+ the initial embedding outputs.
attentions (`tuple(jnp.ndarray)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
+ Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
the self-attention heads.
"""
BEIT_START_DOCSTRING = r"""
- This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading, saving and converting weights from
- PyTorch models)
+ This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading, saving and converting weights from PyTorch models)
- This model is also a Flax Linen [flax.linen.Module](https://flax.readthedocs.io/en/latest/flax.linen.html#module) subclass. Use it as a regular Flax linen Module
- and refer to the Flax documentation for all matter related to general usage and behavior.
+ This model is also a Flax Linen [flax.linen.Module](https://flax.readthedocs.io/en/latest/flax.linen.html#module)
+ subclass. Use it as a regular Flax linen Module and refer to the Flax documentation for all matter related to
+ general usage and behavior.
Finally, this model supports inherent JAX features such as:
@@ -82,11 +83,10 @@ BEIT_START_DOCSTRING = r"""
Parameters:
config ([`BeitConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the
- model weights.
+ configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the model weights.
dtype (`jax.numpy.dtype`, *optional*, defaults to `jax.numpy.float32`):
- The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on
- GPUs) and `jax.numpy.bfloat16` (on TPUs).
+ The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on GPUs) and
+ `jax.numpy.bfloat16` (on TPUs).
This can be used to enable mixed-precision training or half-precision inference on GPUs or TPUs. If
specified all the computation will be performed with the given `dtype`.
@@ -94,8 +94,8 @@ BEIT_START_DOCSTRING = r"""
**Note that this only specifies the dtype of the computation and does not influence the dtype of model
parameters.**
- If you wish to change the dtype of the model parameters, see
- [`~FlaxPreTrainedModel.to_fp16`] and [`~FlaxPreTrainedModel.to_bf16`].
+ If you wish to change the dtype of the model parameters, see [`~FlaxPreTrainedModel.to_fp16`] and
+ [`~FlaxPreTrainedModel.to_bf16`].
"""
BEIT_INPUTS_DOCSTRING = r"""
diff --git a/src/transformers/models/bert/configuration_bert.py b/src/transformers/models/bert/configuration_bert.py
index 885285dfa3..908c6cd432 100644
--- a/src/transformers/models/bert/configuration_bert.py
+++ b/src/transformers/models/bert/configuration_bert.py
@@ -13,7 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" BERT model configuration """
+""" BERT model configuration"""
from collections import OrderedDict
from typing import Mapping
@@ -53,20 +53,19 @@ BERT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class BertConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`BertModel`] or a
- [`TFBertModel`]. It is used to instantiate a BERT model according to the specified arguments,
- defining the model architecture. Instantiating a configuration with the defaults will yield a similar configuration
- to that of the BERT [bert-base-uncased](https://huggingface.co/bert-base-uncased) architecture.
+ This is the configuration class to store the configuration of a [`BertModel`] or a [`TFBertModel`]. It is used to
+ instantiate a BERT model according to the specified arguments, defining the model architecture. Instantiating a
+ configuration with the defaults will yield a similar configuration to that of the BERT
+ [bert-base-uncased](https://huggingface.co/bert-base-uncased) architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
vocab_size (`int`, *optional*, defaults to 30522):
Vocabulary size of the BERT model. Defines the number of different tokens that can be represented by the
- `inputs_ids` passed when calling [`BertModel`] or
- [`TFBertModel`].
+ `inputs_ids` passed when calling [`BertModel`] or [`TFBertModel`].
hidden_size (`int`, *optional*, defaults to 768):
Dimensionality of the encoder layers and the pooler layer.
num_hidden_layers (`int`, *optional*, defaults to 12):
@@ -76,8 +75,8 @@ class BertConfig(PretrainedConfig):
intermediate_size (`int`, *optional*, defaults to 3072):
Dimensionality of the "intermediate" (often named feed-forward) layer in the Transformer encoder.
hidden_act (`str` or `Callable`, *optional*, defaults to `"gelu"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"silu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"silu"` and `"gelu_new"` are supported.
hidden_dropout_prob (`float`, *optional*, defaults to 0.1):
The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
attention_probs_dropout_prob (`float`, *optional*, defaults to 0.1):
@@ -86,17 +85,17 @@ class BertConfig(PretrainedConfig):
The maximum sequence length that this model might ever be used with. Typically set this to something large
just in case (e.g., 512 or 1024 or 2048).
type_vocab_size (`int`, *optional*, defaults to 2):
- The vocabulary size of the `token_type_ids` passed when calling [`BertModel`] or
- [`TFBertModel`].
+ The vocabulary size of the `token_type_ids` passed when calling [`BertModel`] or [`TFBertModel`].
initializer_range (`float`, *optional*, defaults to 0.02):
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
layer_norm_eps (`float`, *optional*, defaults to 1e-12):
The epsilon used by the layer normalization layers.
position_embedding_type (`str`, *optional*, defaults to `"absolute"`):
- Type of position embedding. Choose one of `"absolute"`, `"relative_key"`,
- `"relative_key_query"`. For positional embeddings use `"absolute"`. For more information on
- `"relative_key"`, please refer to [Self-Attention with Relative Position Representations (Shaw et al.)](https://arxiv.org/abs/1803.02155). For more information on `"relative_key_query"`, please refer to
- *Method 4* in [Improve Transformer Models with Better Relative Position Embeddings (Huang et al.)](https://arxiv.org/abs/2009.13658).
+ Type of position embedding. Choose one of `"absolute"`, `"relative_key"`, `"relative_key_query"`. For
+ positional embeddings use `"absolute"`. For more information on `"relative_key"`, please refer to
+ [Self-Attention with Relative Position Representations (Shaw et al.)](https://arxiv.org/abs/1803.02155).
+ For more information on `"relative_key_query"`, please refer to *Method 4* in [Improve Transformer Models
+ with Better Relative Position Embeddings (Huang et al.)](https://arxiv.org/abs/2009.13658).
use_cache (`bool`, *optional*, defaults to `True`):
Whether or not the model should return the last key/values attentions (not used by all models). Only
relevant if `config.is_decoder=True`.
diff --git a/src/transformers/models/bert/modeling_bert.py b/src/transformers/models/bert/modeling_bert.py
index 01bb9fbc17..7eff1dd2e8 100755
--- a/src/transformers/models/bert/modeling_bert.py
+++ b/src/transformers/models/bert/modeling_bert.py
@@ -13,7 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-"""PyTorch BERT model. """
+"""PyTorch BERT model."""
import math
@@ -1130,7 +1130,7 @@ class BertForPreTraining(BertPreTrainedModel):
@add_start_docstrings(
- """Bert Model with a `language modeling` head on top for CLM fine-tuning. """, BERT_START_DOCSTRING
+ """Bert Model with a `language modeling` head on top for CLM fine-tuning.""", BERT_START_DOCSTRING
)
class BertLMHeadModel(BertPreTrainedModel):
@@ -1282,7 +1282,7 @@ class BertLMHeadModel(BertPreTrainedModel):
return reordered_past
-@add_start_docstrings("""Bert Model with a `language modeling` head on top. """, BERT_START_DOCSTRING)
+@add_start_docstrings("""Bert Model with a `language modeling` head on top.""", BERT_START_DOCSTRING)
class BertForMaskedLM(BertPreTrainedModel):
_keys_to_ignore_on_load_unexpected = [r"pooler"]
@@ -1391,7 +1391,7 @@ class BertForMaskedLM(BertPreTrainedModel):
@add_start_docstrings(
- """Bert Model with a `next sentence prediction (classification)` head on top. """,
+ """Bert Model with a `next sentence prediction (classification)` head on top.""",
BERT_START_DOCSTRING,
)
class BertForNextSentencePrediction(BertPreTrainedModel):
diff --git a/src/transformers/models/bert/modeling_flax_bert.py b/src/transformers/models/bert/modeling_flax_bert.py
index 369195df7d..39e334ec8d 100644
--- a/src/transformers/models/bert/modeling_flax_bert.py
+++ b/src/transformers/models/bert/modeling_flax_bert.py
@@ -66,12 +66,13 @@ class FlaxBertForPreTrainingOutput(ModelOutput):
Prediction scores of the next sequence prediction (classification) head (scores of True/False continuation
before SoftMax).
hidden_states (`tuple(jnp.ndarray)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(jnp.ndarray)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -85,12 +86,12 @@ class FlaxBertForPreTrainingOutput(ModelOutput):
BERT_START_DOCSTRING = r"""
- This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading, saving and converting weights from
- PyTorch models)
+ This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading, saving and converting weights from PyTorch models)
- This model is also a Flax Linen [flax.linen.Module](https://flax.readthedocs.io/en/latest/flax.linen.html#module) subclass. Use it as a regular Flax linen Module
- and refer to the Flax documentation for all matter related to general usage and behavior.
+ This model is also a Flax Linen [flax.linen.Module](https://flax.readthedocs.io/en/latest/flax.linen.html#module)
+ subclass. Use it as a regular Flax linen Module and refer to the Flax documentation for all matter related to
+ general usage and behavior.
Finally, this model supports inherent JAX features such as:
@@ -102,11 +103,10 @@ BERT_START_DOCSTRING = r"""
Parameters:
config ([`BertConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the
- model weights.
+ configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the model weights.
dtype (`jax.numpy.dtype`, *optional*, defaults to `jax.numpy.float32`):
- The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on
- GPUs) and `jax.numpy.bfloat16` (on TPUs).
+ The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on GPUs) and
+ `jax.numpy.bfloat16` (on TPUs).
This can be used to enable mixed-precision training or half-precision inference on GPUs or TPUs. If
specified all the computation will be performed with the given `dtype`.
@@ -114,11 +114,11 @@ BERT_START_DOCSTRING = r"""
**Note that this only specifies the dtype of the computation and does not influence the dtype of model
parameters.**
- If you wish to change the dtype of the model parameters, see
- [`~FlaxPreTrainedModel.to_fp16`] and [`~FlaxPreTrainedModel.to_bf16`].
+ If you wish to change the dtype of the model parameters, see [`~FlaxPreTrainedModel.to_fp16`] and
+ [`~FlaxPreTrainedModel.to_bf16`].
dtype (`jax.numpy.dtype`, *optional*, defaults to `jax.numpy.float32`):
- The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on
- GPUs) and `jax.numpy.bfloat16` (on TPUs).
+ The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on GPUs) and
+ `jax.numpy.bfloat16` (on TPUs).
This can be used to enable mixed-precision training or half-precision inference on GPUs or TPUs. If
specified all the computation will be performed with the given `dtype`.
@@ -126,8 +126,8 @@ BERT_START_DOCSTRING = r"""
**Note that this only specifies the dtype of the computation and does not influence the dtype of model
parameters.**
- If you wish to change the dtype of the model parameters, see
- [`~FlaxPreTrainedModel.to_fp16`] and [`~FlaxPreTrainedModel.to_bf16`].
+ If you wish to change the dtype of the model parameters, see [`~FlaxPreTrainedModel.to_fp16`] and
+ [`~FlaxPreTrainedModel.to_bf16`].
"""
@@ -136,9 +136,8 @@ BERT_INPUTS_DOCSTRING = r"""
input_ids (`numpy.ndarray` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`BertTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`BertTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`numpy.ndarray` of shape `({0})`, *optional*):
@@ -149,15 +148,18 @@ BERT_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`numpy.ndarray` of shape `({0})`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`numpy.ndarray` of shape `({0})`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
- head_mask (`numpy.ndarray` of shape `({0})`, `optional): Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`:
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
+ head_mask (`numpy.ndarray` of shape `({0})`, `optional):
+ Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`:
- 1 indicates the head is **not masked**,
- 0 indicates the head is **masked**.
@@ -909,7 +911,7 @@ class FlaxBertForMaskedLMModule(nn.Module):
)
-@add_start_docstrings("""Bert Model with a `language modeling` head on top. """, BERT_START_DOCSTRING)
+@add_start_docstrings("""Bert Model with a `language modeling` head on top.""", BERT_START_DOCSTRING)
class FlaxBertForMaskedLM(FlaxBertPreTrainedModel):
module_class = FlaxBertForMaskedLMModule
@@ -968,7 +970,7 @@ class FlaxBertForNextSentencePredictionModule(nn.Module):
@add_start_docstrings(
- """Bert Model with a `next sentence prediction (classification)` head on top. """,
+ """Bert Model with a `next sentence prediction (classification)` head on top.""",
BERT_START_DOCSTRING,
)
class FlaxBertForNextSentencePrediction(FlaxBertPreTrainedModel):
diff --git a/src/transformers/models/bert/modeling_tf_bert.py b/src/transformers/models/bert/modeling_tf_bert.py
index 3b3854be71..7f23abdc8c 100644
--- a/src/transformers/models/bert/modeling_tf_bert.py
+++ b/src/transformers/models/bert/modeling_tf_bert.py
@@ -13,7 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" TF 2.0 BERT model. """
+""" TF 2.0 BERT model."""
import math
import warnings
@@ -938,12 +938,13 @@ class TFBertForPreTrainingOutput(ModelOutput):
Prediction scores of the next sequence prediction (classification) head (scores of True/False continuation
before SoftMax).
hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -958,13 +959,13 @@ class TFBertForPreTrainingOutput(ModelOutput):
BERT_START_DOCSTRING = r"""
- This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading or saving, resizing the input
- embeddings, pruning heads etc.)
+ This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use
- it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage
- and behavior.
+ This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it
+ as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and
+ behavior.
@@ -973,11 +974,11 @@ BERT_START_DOCSTRING = r"""
- having all inputs as keyword arguments (like PyTorch models), or
- having all inputs as a list, tuple or dict in the first positional arguments.
- This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all
- the tensors in the first argument of the model call function: `model(inputs)`.
+ This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all the
+ tensors in the first argument of the model call function: `model(inputs)`.
- If you choose this second option, there are three possibilities you can use to gather all the input Tensors in
- the first positional argument :
+ If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the
+ first positional argument :
- a single Tensor with `input_ids` only and nothing else: `model(inputs_ids)`
- a list of varying length with one or several input Tensors IN THE ORDER given in the docstring:
@@ -990,8 +991,7 @@ BERT_START_DOCSTRING = r"""
Args:
config ([`BertConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~TFPreTrainedModel.from_pretrained`] method to load the
- model weights.
+ configuration. Check out the [`~TFPreTrainedModel.from_pretrained`] method to load the model weights.
"""
BERT_INPUTS_DOCSTRING = r"""
@@ -999,9 +999,8 @@ BERT_INPUTS_DOCSTRING = r"""
input_ids (`np.ndarray`, `tf.Tensor`, `List[tf.Tensor]` ``Dict[str, tf.Tensor]` or `Dict[str, np.ndarray]` and each example must have the shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`BertTokenizer`]. See
- [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for
- details.
+ Indices can be obtained using [`BertTokenizer`]. See [`PreTrainedTokenizer.__call__`] and
+ [`PreTrainedTokenizer.encode`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`np.ndarray` or `tf.Tensor` of shape `({0})`, *optional*):
@@ -1012,14 +1011,16 @@ BERT_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`np.ndarray` or `tf.Tensor` of shape `({0})`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`np.ndarray` or `tf.Tensor` of shape `({0})`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
head_mask (`np.ndarray` or `tf.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
@@ -1029,9 +1030,9 @@ BERT_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
inputs_embeds (`np.ndarray` or `tf.Tensor` of shape `({0}, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the
@@ -1041,8 +1042,8 @@ BERT_INPUTS_DOCSTRING = r"""
more detail. This argument can be used only in eager mode, in graph mode the value in the config will be
used instead.
return_dict (`bool`, *optional*):
- Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This
- argument can be used in eager mode, in graph mode the value will always be set to True.
+ Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This argument can be used
+ in eager mode, in graph mode the value will always be set to True.
training (`bool`, *optional*, defaults to `False``):
Whether or not to use the model in training mode (some modules like dropout modules have different
behaviors between training and evaluation).
@@ -1097,12 +1098,12 @@ class TFBertModel(TFBertPreTrainedModel):
past_key_values (`Tuple[Tuple[tf.Tensor]]` of length `config.n_layers`)
contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ `decoder_input_ids` of shape `(batch_size, sequence_length)`.
use_cache (`bool`, *optional*, defaults to `True`):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`). Set to `False` during training, `True` during generation
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`). Set to `False` during training, `True` during generation
"""
inputs = input_processing(
func=self.call,
@@ -1212,8 +1213,9 @@ class TFBertForPreTraining(TFBertPreTrainedModel, TFBertPreTrainingLoss):
) -> Union[TFBertForPreTrainingOutput, Tuple[tf.Tensor]]:
r"""
labels (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
next_sentence_label (`tf.Tensor` of shape `(batch_size,)`, *optional*):
Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair
(see `input_ids` docstring) Indices should be in `[0, 1]`:
@@ -1300,7 +1302,7 @@ class TFBertForPreTraining(TFBertPreTrainedModel, TFBertPreTrainingLoss):
)
-@add_start_docstrings("""Bert Model with a `language modeling` head on top. """, BERT_START_DOCSTRING)
+@add_start_docstrings("""Bert Model with a `language modeling` head on top.""", BERT_START_DOCSTRING)
class TFBertForMaskedLM(TFBertPreTrainedModel, TFMaskedLanguageModelingLoss):
# names with a '.' represents the authorized unexpected/missing layers when a TF model is loaded from a PT model
_keys_to_ignore_on_load_unexpected = [
@@ -1353,8 +1355,9 @@ class TFBertForMaskedLM(TFBertPreTrainedModel, TFMaskedLanguageModelingLoss):
) -> Union[TFMaskedLMOutput, Tuple[tf.Tensor]]:
r"""
labels (`tf.Tensor` or `np.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
"""
inputs = input_processing(
func=self.call,
@@ -1483,14 +1486,15 @@ class TFBertLMHeadModel(TFBertPreTrainedModel, TFCausalLanguageModelingLoss):
past_key_values (`Tuple[Tuple[tf.Tensor]]` of length `config.n_layers`)
contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ `decoder_input_ids` of shape `(batch_size, sequence_length)`.
use_cache (`bool`, *optional*, defaults to `True`):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`). Set to `False` during training, `True` during generation
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`). Set to `False` during training, `True` during generation
labels (`tf.Tensor` or `np.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the cross entropy classification loss. Indices should be in `[0, ..., config.vocab_size - 1]`.
+ Labels for computing the cross entropy classification loss. Indices should be in `[0, ...,
+ config.vocab_size - 1]`.
"""
inputs = input_processing(
func=self.call,
@@ -1566,7 +1570,7 @@ class TFBertLMHeadModel(TFBertPreTrainedModel, TFCausalLanguageModelingLoss):
@add_start_docstrings(
- """Bert Model with a `next sentence prediction (classification)` head on top. """,
+ """Bert Model with a `next sentence prediction (classification)` head on top.""",
BERT_START_DOCSTRING,
)
class TFBertForNextSentencePrediction(TFBertPreTrainedModel, TFNextSentencePredictionLoss):
@@ -1721,8 +1725,9 @@ class TFBertForSequenceClassification(TFBertPreTrainedModel, TFSequenceClassific
) -> Union[TFSequenceClassifierOutput, Tuple[tf.Tensor]]:
r"""
labels (`tf.Tensor` or `np.ndarray` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
inputs = input_processing(
func=self.call,
@@ -1830,8 +1835,8 @@ class TFBertForMultipleChoice(TFBertPreTrainedModel, TFMultipleChoiceLoss):
) -> Union[TFMultipleChoiceModelOutput, Tuple[tf.Tensor]]:
r"""
labels (`tf.Tensor` or `np.ndarray` of shape `(batch_size,)`, *optional*):
- Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices]` where `num_choices` is the size of the second dimension of the input tensors. (See
- `input_ids` above)
+ Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices]`
+ where `num_choices` is the size of the second dimension of the input tensors. (See `input_ids` above)
"""
inputs = input_processing(
func=self.call,
@@ -2096,12 +2101,12 @@ class TFBertForQuestionAnswering(TFBertPreTrainedModel, TFQuestionAnsweringLoss)
r"""
start_positions (`tf.Tensor` or `np.ndarray` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the start of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
end_positions (`tf.Tensor` or `np.ndarray` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the end of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
"""
inputs = input_processing(
func=self.call,
diff --git a/src/transformers/models/bert/tokenization_bert.py b/src/transformers/models/bert/tokenization_bert.py
index 5520f845cc..7fa7be4878 100644
--- a/src/transformers/models/bert/tokenization_bert.py
+++ b/src/transformers/models/bert/tokenization_bert.py
@@ -118,8 +118,8 @@ class BertTokenizer(PreTrainedTokenizer):
r"""
Construct a BERT tokenizer. Based on WordPiece.
- This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods.
- Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
+ this superclass for more information regarding those methods.
Args:
vocab_file (`str`):
@@ -149,7 +149,8 @@ class BertTokenizer(PreTrainedTokenizer):
tokenize_chinese_chars (`bool`, *optional*, defaults to `True`):
Whether or not to tokenize Chinese characters.
- This should likely be deactivated for Japanese (see this [issue](https://github.com/huggingface/transformers/issues/328)).
+ This should likely be deactivated for Japanese (see this
+ [issue](https://github.com/huggingface/transformers/issues/328)).
strip_accents: (`bool`, *optional*):
Whether or not to strip all accents. If this option is not specified, then it will be determined by the
value for `lowercase` (as in the original BERT).
@@ -318,8 +319,7 @@ class BertTokenizer(PreTrainedTokenizer):
Optional second list of IDs for sequence pairs.
Returns:
- `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given
- sequence(s).
+ `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
"""
sep = [self.sep_token_id]
cls = [self.cls_token_id]
@@ -361,7 +361,8 @@ class BasicTokenizer(object):
tokenize_chinese_chars (`bool`, *optional*, defaults to `True`):
Whether or not to tokenize Chinese characters.
- This should likely be deactivated for Japanese (see this [issue](https://github.com/huggingface/transformers/issues/328)).
+ This should likely be deactivated for Japanese (see this
+ [issue](https://github.com/huggingface/transformers/issues/328)).
strip_accents: (`bool`, *optional*):
Whether or not to strip all accents. If this option is not specified, then it will be determined by the
value for `lowercase` (as in the original BERT).
diff --git a/src/transformers/models/bert/tokenization_bert_fast.py b/src/transformers/models/bert/tokenization_bert_fast.py
index 5b0ebaf086..4fd53be98d 100644
--- a/src/transformers/models/bert/tokenization_bert_fast.py
+++ b/src/transformers/models/bert/tokenization_bert_fast.py
@@ -118,8 +118,8 @@ class BertTokenizerFast(PreTrainedTokenizerFast):
r"""
Construct a "fast" BERT tokenizer (backed by HuggingFace's *tokenizers* library). Based on WordPiece.
- This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main
- methods. Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should
+ refer to this superclass for more information regarding those methods.
Args:
vocab_file (`str`):
@@ -245,8 +245,7 @@ class BertTokenizerFast(PreTrainedTokenizerFast):
Optional second list of IDs for sequence pairs.
Returns:
- `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given
- sequence(s).
+ `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
"""
sep = [self.sep_token_id]
cls = [self.cls_token_id]
diff --git a/src/transformers/models/bert_generation/configuration_bert_generation.py b/src/transformers/models/bert_generation/configuration_bert_generation.py
index 3c79f25fd2..effe415ee0 100644
--- a/src/transformers/models/bert_generation/configuration_bert_generation.py
+++ b/src/transformers/models/bert_generation/configuration_bert_generation.py
@@ -12,19 +12,18 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" BertGeneration model configuration """
+""" BertGeneration model configuration"""
from ...configuration_utils import PretrainedConfig
class BertGenerationConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a
- [`BertGenerationPreTrainedModel`]. It is used to instantiate a BertGeneration model according to
- the specified arguments, defining the model architecture.
+ This is the configuration class to store the configuration of a [`BertGenerationPreTrainedModel`]. It is used to
+ instantiate a BertGeneration model according to the specified arguments, defining the model architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
vocab_size (`int`, *optional*, defaults to 50358):
@@ -39,8 +38,8 @@ class BertGenerationConfig(PretrainedConfig):
intermediate_size (`int`, *optional*, defaults to 3072):
Dimensionality of the "intermediate" (often called feed-forward) layer in the Transformer encoder.
hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"silu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"silu"` and `"gelu_new"` are supported.
hidden_dropout_prob (`float`, *optional*, defaults to 0.1):
The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
attention_probs_dropout_prob (`float`, *optional*, defaults to 0.1):
@@ -53,10 +52,11 @@ class BertGenerationConfig(PretrainedConfig):
layer_norm_eps (`float`, *optional*, defaults to 1e-12):
The epsilon used by the layer normalization layers.
position_embedding_type (`str`, *optional*, defaults to `"absolute"`):
- Type of position embedding. Choose one of `"absolute"`, `"relative_key"`,
- `"relative_key_query"`. For positional embeddings use `"absolute"`. For more information on
- `"relative_key"`, please refer to [Self-Attention with Relative Position Representations (Shaw et al.)](https://arxiv.org/abs/1803.02155). For more information on `"relative_key_query"`, please refer to
- *Method 4* in [Improve Transformer Models with Better Relative Position Embeddings (Huang et al.)](https://arxiv.org/abs/2009.13658).
+ Type of position embedding. Choose one of `"absolute"`, `"relative_key"`, `"relative_key_query"`. For
+ positional embeddings use `"absolute"`. For more information on `"relative_key"`, please refer to
+ [Self-Attention with Relative Position Representations (Shaw et al.)](https://arxiv.org/abs/1803.02155).
+ For more information on `"relative_key_query"`, please refer to *Method 4* in [Improve Transformer Models
+ with Better Relative Position Embeddings (Huang et al.)](https://arxiv.org/abs/2009.13658).
use_cache (`bool`, *optional*, defaults to `True`):
Whether or not the model should return the last key/values attentions (not used by all models). Only
relevant if `config.is_decoder=True`.
diff --git a/src/transformers/models/bert_generation/modeling_bert_generation.py b/src/transformers/models/bert_generation/modeling_bert_generation.py
index b126fae2e4..0ec8ab9705 100755
--- a/src/transformers/models/bert_generation/modeling_bert_generation.py
+++ b/src/transformers/models/bert_generation/modeling_bert_generation.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-"""PyTorch BERT model specific for generation. """
+"""PyTorch BERT model specific for generation."""
import torch
@@ -195,19 +195,18 @@ class BertGenerationPreTrainedModel(PreTrainedModel):
BERT_GENERATION_START_DOCSTRING = r"""
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
Parameters:
config ([`BertGenerationConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
BERT_GENERATION_INPUTS_DOCSTRING = r"""
@@ -215,9 +214,8 @@ BERT_GENERATION_INPUTS_DOCSTRING = r"""
input_ids (`torch.LongTensor` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`BertGenerationTokenizer`]. See
- [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for
- details.
+ Indices can be obtained using [`BertGenerationTokenizer`]. See [`PreTrainedTokenizer.__call__`] and
+ [`PreTrainedTokenizer.encode`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
@@ -228,7 +226,8 @@ BERT_GENERATION_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
@@ -238,9 +237,9 @@ BERT_GENERATION_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -264,14 +263,13 @@ class BertGenerationEncoder(BertGenerationPreTrainedModel):
all you need](https://arxiv.org/abs/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit,
Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin.
- This model should be used when leveraging Bert or Roberta checkpoints for the
- [`EncoderDecoderModel`] class as described in [Leveraging Pre-trained Checkpoints for Sequence
- Generation Tasks](https://arxiv.org/abs/1907.12461) by Sascha Rothe, Shashi Narayan, and Aliaksei Severyn.
+ This model should be used when leveraging Bert or Roberta checkpoints for the [`EncoderDecoderModel`] class as
+ described in [Leveraging Pre-trained Checkpoints for Sequence Generation Tasks](https://arxiv.org/abs/1907.12461)
+ by Sascha Rothe, Shashi Narayan, and Aliaksei Severyn.
- To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration
- set to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder`
- argument and `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an
- input to the forward pass.
+ To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set
+ to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and
+ `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass.
"""
def __init__(self, config):
@@ -331,12 +329,12 @@ class BertGenerationEncoder(BertGenerationPreTrainedModel):
past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ `decoder_input_ids` of shape `(batch_size, sequence_length)`.
use_cache (`bool`, *optional*):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`).
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`).
"""
output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
output_hidden_states = (
@@ -443,7 +441,7 @@ class BertGenerationOnlyLMHead(nn.Module):
@add_start_docstrings(
- """BertGeneration Model with a `language modeling` head on top for CLM fine-tuning. """,
+ """BertGeneration Model with a `language modeling` head on top for CLM fine-tuning.""",
BERT_GENERATION_START_DOCSTRING,
)
class BertGenerationDecoder(BertGenerationPreTrainedModel):
@@ -500,12 +498,12 @@ class BertGenerationDecoder(BertGenerationPreTrainedModel):
past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ `decoder_input_ids` of shape `(batch_size, sequence_length)`.
use_cache (`bool`, *optional*):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`).
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`).
Returns:
diff --git a/src/transformers/models/bert_generation/tokenization_bert_generation.py b/src/transformers/models/bert_generation/tokenization_bert_generation.py
index f6b7a7f9cc..66da1f0658 100644
--- a/src/transformers/models/bert_generation/tokenization_bert_generation.py
+++ b/src/transformers/models/bert_generation/tokenization_bert_generation.py
@@ -42,8 +42,8 @@ class BertGenerationTokenizer(PreTrainedTokenizer):
"""
Construct a BertGeneration tokenizer. Based on [SentencePiece](https://github.com/google/sentencepiece).
- This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods.
- Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
+ this superclass for more information regarding those methods.
Args:
vocab_file (`str`):
@@ -59,7 +59,9 @@ class BertGenerationTokenizer(PreTrainedTokenizer):
pad_token (`str`, *optional*, defaults to `""`):
The token used for padding, for example when batching sequences of different lengths.
sp_model_kwargs (`dict`, *optional*):
- Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things, to set:
+ Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for
+ SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things,
+ to set:
- `enable_sampling`: Enable subword regularization.
- `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout.
diff --git a/src/transformers/models/bertweet/tokenization_bertweet.py b/src/transformers/models/bertweet/tokenization_bertweet.py
index ee1e3bd7d7..f445b68ed7 100644
--- a/src/transformers/models/bertweet/tokenization_bertweet.py
+++ b/src/transformers/models/bertweet/tokenization_bertweet.py
@@ -13,7 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" Tokenization classes for BERTweet """
+""" Tokenization classes for BERTweet"""
import html
@@ -69,8 +69,8 @@ class BertweetTokenizer(PreTrainedTokenizer):
"""
Constructs a BERTweet tokenizer, using Byte-Pair-Encoding.
- This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods.
- Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
+ this superclass for more information regarding those methods.
Args:
vocab_file (`str`):
@@ -94,8 +94,8 @@ class BertweetTokenizer(PreTrainedTokenizer):
- When building a sequence using special tokens, this is not the token that is used for the end of
- sequence. The token used is the `sep_token`.
+ When building a sequence using special tokens, this is not the token that is used for the end of sequence.
+ The token used is the `sep_token`.
diff --git a/src/transformers/models/big_bird/configuration_big_bird.py b/src/transformers/models/big_bird/configuration_big_bird.py
index 80dd708b92..eac6aff79d 100644
--- a/src/transformers/models/big_bird/configuration_big_bird.py
+++ b/src/transformers/models/big_bird/configuration_big_bird.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" BigBird model configuration """
+""" BigBird model configuration"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging
@@ -30,13 +30,13 @@ BIG_BIRD_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class BigBirdConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`BigBirdModel`]. It is used to
- instantiate an BigBird model according to the specified arguments, defining the model architecture. Instantiating a
- configuration with the defaults will yield a similar configuration to that of the BigBird
+ This is the configuration class to store the configuration of a [`BigBirdModel`]. It is used to instantiate an
+ BigBird model according to the specified arguments, defining the model architecture. Instantiating a configuration
+ with the defaults will yield a similar configuration to that of the BigBird
[google/bigbird-roberta-base](https://huggingface.co/google/bigbird-roberta-base) architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
@@ -52,8 +52,8 @@ class BigBirdConfig(PretrainedConfig):
intermediate_size (`int`, *optional*, defaults to 3072):
Dimension of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
hidden_act (`str` or `function`, *optional*, defaults to `"gelu_new"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"selu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"selu"` and `"gelu_new"` are supported.
hidden_dropout_prob (`float`, *optional*, defaults to 0.1):
The dropout probabilitiy for all fully connected layers in the embeddings, encoder, and pooler.
attention_probs_dropout_prob (`float`, *optional*, defaults to 0.1):
@@ -80,7 +80,8 @@ class BigBirdConfig(PretrainedConfig):
block_size (`int`, *optional*, defaults to 64)
Size of each block. Useful only when `attention_type == "block_sparse"`.
num_random_blocks (`int`, *optional*, defaults to 3)
- Each query is going to attend these many number of random blocks. Useful only when `attention_type == "block_sparse"`.
+ Each query is going to attend these many number of random blocks. Useful only when `attention_type ==
+ "block_sparse"`.
classifier_dropout (`float`, *optional*):
The dropout ratio for the classification head.
@@ -92,14 +93,13 @@ class BigBirdConfig(PretrainedConfig):
>>> from transformers import BigBirdModel, BigBirdConfig
- >>> # Initializing a BigBird google/bigbird-roberta-base style configuration
- >>> configuration = BigBirdConfig()
+ >>> # Initializing a BigBird google/bigbird-roberta-base style configuration >>> configuration =
+ BigBirdConfig()
- >>> # Initializing a model from the google/bigbird-roberta-base style configuration
- >>> model = BigBirdModel(configuration)
+ >>> # Initializing a model from the google/bigbird-roberta-base style configuration >>> model =
+ BigBirdModel(configuration)
- >>> # Accessing the model configuration
- >>> configuration = model.config
+ >>> # Accessing the model configuration >>> configuration = model.config
"""
model_type = "big_bird"
diff --git a/src/transformers/models/big_bird/modeling_big_bird.py b/src/transformers/models/big_bird/modeling_big_bird.py
index 3699aa3a63..47b9644430 100755
--- a/src/transformers/models/big_bird/modeling_big_bird.py
+++ b/src/transformers/models/big_bird/modeling_big_bird.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" PyTorch BigBird model. """
+""" PyTorch BigBird model."""
import math
@@ -1788,8 +1788,7 @@ BIG_BIRD_START_DOCSTRING = r"""
Parameters:
config ([`BigBirdConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
BIG_BIRD_INPUTS_DOCSTRING = r"""
@@ -1797,9 +1796,8 @@ BIG_BIRD_INPUTS_DOCSTRING = r"""
input_ids (`torch.LongTensor` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`BigBirdTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`BigBirdTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
@@ -1810,14 +1808,16 @@ BIG_BIRD_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
@@ -1827,9 +1827,9 @@ BIG_BIRD_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert *input_ids* indices into associated vectors
- than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -1856,12 +1856,13 @@ class BigBirdForPreTrainingOutput(ModelOutput):
Prediction scores of the next sequence prediction (classification) head (scores of True/False continuation
before SoftMax).
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -1889,12 +1890,13 @@ class BigBirdForQuestionAnsweringModelOutput(ModelOutput):
pooler_output (`torch.FloatTensor` of shape `(batch_size, 1)`):
pooler output from BigBigModel
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -1920,10 +1922,9 @@ class BigBirdModel(BigBirdPreTrainedModel):
all you need](https://arxiv.org/abs/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit,
Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin.
- To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration
- set to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder`
- argument and `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an
- input to the forward pass.
+ To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set
+ to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and
+ `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass.
"""
def __init__(self, config, add_pooling_layer=True):
@@ -2004,12 +2005,12 @@ class BigBirdModel(BigBirdPreTrainedModel):
- 0 for tokens that are **masked**.
past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ `decoder_input_ids` of shape `(batch_size, sequence_length)`.
use_cache (`bool`, *optional*):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`).
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`).
"""
output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
output_hidden_states = (
@@ -2286,12 +2287,13 @@ class BigBirdForPreTraining(BigBirdPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
next_sentence_label (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for computing the next sequence prediction (classification) loss. If specified, nsp loss will be
- added to masked_lm loss. Input should be a sequence pair (see `input_ids` docstring) Indices should be
- in `[0, 1]`:
+ added to masked_lm loss. Input should be a sequence pair (see `input_ids` docstring) Indices should be in
+ `[0, 1]`:
- 0 indicates sequence B is a continuation of sequence A,
- 1 indicates sequence B is a random sequence.
@@ -2354,7 +2356,7 @@ class BigBirdForPreTraining(BigBirdPreTrainedModel):
)
-@add_start_docstrings("""BigBird Model with a `language modeling` head on top. """, BIG_BIRD_START_DOCSTRING)
+@add_start_docstrings("""BigBird Model with a `language modeling` head on top.""", BIG_BIRD_START_DOCSTRING)
class BigBirdForMaskedLM(BigBirdPreTrainedModel):
def __init__(self, config):
super().__init__(config)
@@ -2401,8 +2403,9 @@ class BigBirdForMaskedLM(BigBirdPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -2455,7 +2458,7 @@ class BigBirdForMaskedLM(BigBirdPreTrainedModel):
@add_start_docstrings(
- """BigBird Model with a `language modeling` head on top for CLM fine-tuning. """, BIG_BIRD_START_DOCSTRING
+ """BigBird Model with a `language modeling` head on top for CLM fine-tuning.""", BIG_BIRD_START_DOCSTRING
)
class BigBirdForCausalLM(BigBirdPreTrainedModel):
@@ -2510,16 +2513,16 @@ class BigBirdForCausalLM(BigBirdPreTrainedModel):
- 0 for tokens that are **masked**.
past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ `decoder_input_ids` of shape `(batch_size, sequence_length)`.
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in
`[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are
ignored (masked), the loss is only computed for the tokens with labels n `[0, ..., config.vocab_size]`.
use_cache (`bool`, *optional*):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`).
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`).
Returns:
@@ -2667,8 +2670,9 @@ class BigBirdForSequenceClassification(BigBirdPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -2764,7 +2768,8 @@ class BigBirdForMultipleChoice(BigBirdPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
+ Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
+ num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
`input_ids` above)
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -2970,12 +2975,12 @@ class BigBirdForQuestionAnswering(BigBirdPreTrainedModel):
r"""
start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the start of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the end of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
diff --git a/src/transformers/models/big_bird/modeling_flax_big_bird.py b/src/transformers/models/big_bird/modeling_flax_big_bird.py
index a1be468934..214eb9740f 100644
--- a/src/transformers/models/big_bird/modeling_flax_big_bird.py
+++ b/src/transformers/models/big_bird/modeling_flax_big_bird.py
@@ -64,12 +64,13 @@ class FlaxBigBirdForPreTrainingOutput(ModelOutput):
Prediction scores of the next sequence prediction (classification) head (scores of True/False continuation
before SoftMax).
hidden_states (`tuple(jnp.ndarray)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(jnp.ndarray)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -94,12 +95,13 @@ class FlaxBigBirdForQuestionAnsweringModelOutput(ModelOutput):
pooled_output (`jnp.ndarray` of shape `(batch_size, hidden_size)`):
pooled_output returned by FlaxBigBirdModel.
hidden_states (`tuple(jnp.ndarray)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(jnp.ndarray)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -114,12 +116,12 @@ class FlaxBigBirdForQuestionAnsweringModelOutput(ModelOutput):
BIG_BIRD_START_DOCSTRING = r"""
- This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading, saving and converting weights from
- PyTorch models)
+ This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading, saving and converting weights from PyTorch models)
- This model is also a Flax Linen [flax.linen.Module](https://flax.readthedocs.io/en/latest/flax.linen.html#module) subclass. Use it as a regular Flax linen Module
- and refer to the Flax documentation for all matter related to general usage and behavior.
+ This model is also a Flax Linen [flax.linen.Module](https://flax.readthedocs.io/en/latest/flax.linen.html#module)
+ subclass. Use it as a regular Flax linen Module and refer to the Flax documentation for all matter related to
+ general usage and behavior.
Finally, this model supports inherent JAX features such as:
@@ -131,11 +133,10 @@ BIG_BIRD_START_DOCSTRING = r"""
Parameters:
config ([`BigBirdConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the
- model weights.
+ configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the model weights.
dtype (`jax.numpy.dtype`, *optional*, defaults to `jax.numpy.float32`):
- The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on
- GPUs) and `jax.numpy.bfloat16` (on TPUs).
+ The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on GPUs) and
+ `jax.numpy.bfloat16` (on TPUs).
This can be used to enable mixed-precision training or half-precision inference on GPUs or TPUs. If
specified all the computation will be performed with the given `dtype`.
@@ -143,8 +144,8 @@ BIG_BIRD_START_DOCSTRING = r"""
**Note that this only specifies the dtype of the computation and does not influence the dtype of model
parameters.**
- If you wish to change the dtype of the model parameters, see
- [`~FlaxPreTrainedModel.to_fp16`] and [`~FlaxPreTrainedModel.to_bf16`].
+ If you wish to change the dtype of the model parameters, see [`~FlaxPreTrainedModel.to_fp16`] and
+ [`~FlaxPreTrainedModel.to_bf16`].
"""
BIG_BIRD_INPUTS_DOCSTRING = r"""
@@ -152,9 +153,8 @@ BIG_BIRD_INPUTS_DOCSTRING = r"""
input_ids (`numpy.ndarray` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`BigBirdTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`BigBirdTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`numpy.ndarray` of shape `({0})`, *optional*):
@@ -165,15 +165,18 @@ BIG_BIRD_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`numpy.ndarray` of shape `({0})`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`numpy.ndarray` of shape `({0})`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
- head_mask (`numpy.ndarray` of shape `({0})`, `optional): Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`:
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
+ head_mask (`numpy.ndarray` of shape `({0})`, `optional):
+ Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`:
- 1 indicates the head is **not masked**,
- 0 indicates the head is **masked**.
@@ -787,7 +790,8 @@ class FlaxBigBirdBlockSparseAttention(nn.Module):
Args:
from_blocked_mask: 2D Tensor of shape [batch_size, from_seq_length//from_block_size, from_block_size].
to_blocked_mask: int32 Tensor of shape [batch_size, to_seq_length//to_block_size, to_block_size].
- broadcasted_rand_attn: [batch_size, num_attention_heads, from_seq_length//from_block_size-2, num_rand_blocks]
+ broadcasted_rand_attn:
+ [batch_size, num_attention_heads, from_seq_length//from_block_size-2, num_rand_blocks]
num_attention_heads: int. Number of attention heads.
num_random_blocks: int. Number of random chunks per row.
batch_size: int. Batch size for computation.
@@ -1713,7 +1717,7 @@ class FlaxBigBirdForMaskedLMModule(nn.Module):
)
-@add_start_docstrings("""BigBird Model with a `language modeling` head on top. """, BIG_BIRD_START_DOCSTRING)
+@add_start_docstrings("""BigBird Model with a `language modeling` head on top.""", BIG_BIRD_START_DOCSTRING)
# Copied from transformers.models.bert.modeling_flax_bert.FlaxBertForMaskedLM with Bert->BigBird
class FlaxBigBirdForMaskedLM(FlaxBigBirdPreTrainedModel):
module_class = FlaxBigBirdForMaskedLMModule
diff --git a/src/transformers/models/big_bird/tokenization_big_bird.py b/src/transformers/models/big_bird/tokenization_big_bird.py
index 355e3fd068..536fdc0b1d 100644
--- a/src/transformers/models/big_bird/tokenization_big_bird.py
+++ b/src/transformers/models/big_bird/tokenization_big_bird.py
@@ -48,8 +48,8 @@ class BigBirdTokenizer(PreTrainedTokenizer):
"""
Construct a BigBird tokenizer. Based on [SentencePiece](https://github.com/google/sentencepiece).
- This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods.
- Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
+ this superclass for more information regarding those methods.
Args:
vocab_file (`str`):
@@ -75,7 +75,9 @@ class BigBirdTokenizer(PreTrainedTokenizer):
The token used for masking values. This is the token used when training this model with masked language
modeling. This is the token which the model will try to predict.
sp_model_kwargs (`dict`, *optional*):
- Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things, to set:
+ Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for
+ SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things,
+ to set:
- `enable_sampling`: Enable subword regularization.
- `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout.
@@ -259,8 +261,7 @@ class BigBirdTokenizer(PreTrainedTokenizer):
Optional second list of IDs for sequence pairs.
Returns:
- `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given
- sequence(s).
+ `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
"""
sep = [self.sep_token_id]
cls = [self.cls_token_id]
diff --git a/src/transformers/models/big_bird/tokenization_big_bird_fast.py b/src/transformers/models/big_bird/tokenization_big_bird_fast.py
index 26adf8a3ec..3d0aa94627 100644
--- a/src/transformers/models/big_bird/tokenization_big_bird_fast.py
+++ b/src/transformers/models/big_bird/tokenization_big_bird_fast.py
@@ -58,9 +58,10 @@ SPIECE_UNDERLINE = "▁"
class BigBirdTokenizerFast(PreTrainedTokenizerFast):
"""
- Construct a "fast" BigBird tokenizer (backed by HuggingFace's *tokenizers* library). Based on [Unigram](https://huggingface.co/docs/tokenizers/python/latest/components.html?highlight=unigram#models). This tokenizer
- inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should
- refer to this superclass for more information regarding those methods
+ Construct a "fast" BigBird tokenizer (backed by HuggingFace's *tokenizers* library). Based on
+ [Unigram](https://huggingface.co/docs/tokenizers/python/latest/components.html?highlight=unigram#models). This
+ tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should refer to
+ this superclass for more information regarding those methods
Args:
vocab_file (`str`):
@@ -219,8 +220,7 @@ class BigBirdTokenizerFast(PreTrainedTokenizerFast):
Optional second list of IDs for sequence pairs.
Returns:
- `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given
- sequence(s).
+ `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
"""
sep = [self.sep_token_id]
cls = [self.cls_token_id]
diff --git a/src/transformers/models/bigbird_pegasus/configuration_bigbird_pegasus.py b/src/transformers/models/bigbird_pegasus/configuration_bigbird_pegasus.py
index 2d9fdd18d7..7c9c0ec5a0 100644
--- a/src/transformers/models/bigbird_pegasus/configuration_bigbird_pegasus.py
+++ b/src/transformers/models/bigbird_pegasus/configuration_bigbird_pegasus.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" BigBirdPegasus model configuration """
+""" BigBirdPegasus model configuration"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging
@@ -30,13 +30,13 @@ BIGBIRD_PEGASUS_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class BigBirdPegasusConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`BigBirdPegasusModel`]. It is
- used to instantiate an BigBirdPegasus model according to the specified arguments, defining the model architecture.
- Instantiating a configuration with the defaults will yield a similar configuration to that of the BigBirdPegasus
+ This is the configuration class to store the configuration of a [`BigBirdPegasusModel`]. It is used to instantiate
+ an BigBirdPegasus model according to the specified arguments, defining the model architecture. Instantiating a
+ configuration with the defaults will yield a similar configuration to that of the BigBirdPegasus
[google/bigbird-pegasus-large-arxiv](https://huggingface.co/google/bigbird-pegasus-large-arxiv) architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
@@ -58,8 +58,8 @@ class BigBirdPegasusConfig(PretrainedConfig):
encoder_ffn_dim (`int`, *optional*, defaults to 4096):
Dimension of the "intermediate" (often named feed-forward) layer in decoder.
activation_function (`str` or `function`, *optional*, defaults to `"gelu_new"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"silu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"silu"` and `"gelu_new"` are supported.
dropout (`float`, *optional*, defaults to 0.1):
The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
attention_dropout (`float`, *optional*, defaults to 0.0):
@@ -74,23 +74,23 @@ class BigBirdPegasusConfig(PretrainedConfig):
init_std (`float`, *optional*, defaults to 0.02):
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
encoder_layerdrop: (`float`, *optional*, defaults to 0.0):
- The LayerDrop probability for the encoder. See the [LayerDrop paper](see
- https://arxiv.org/abs/1909.11556) for more details.
+ The LayerDrop probability for the encoder. See the [LayerDrop paper](see https://arxiv.org/abs/1909.11556)
+ for more details.
decoder_layerdrop: (`float`, *optional*, defaults to 0.0):
- The LayerDrop probability for the decoder. See the [LayerDrop paper](see
- https://arxiv.org/abs/1909.11556) for more details.
+ The LayerDrop probability for the decoder. See the [LayerDrop paper](see https://arxiv.org/abs/1909.11556)
+ for more details.
use_cache (`bool`, *optional*, defaults to `True`):
Whether or not the model should return the last key/values attentions (not used by all models).
attention_type (`str`, *optional*, defaults to `"block_sparse"`)
Whether to use block sparse attention (with n complexity) as introduced in paper or original attention
- layer (with n^2 complexity) in encoder. Possible values are `"original_full"` and
- `"block_sparse"`.
+ layer (with n^2 complexity) in encoder. Possible values are `"original_full"` and `"block_sparse"`.
use_bias (`bool`, *optional*, defaults to `False`)
Whether to use bias in query, key, value.
block_size (`int`, *optional*, defaults to 64)
Size of each block. Useful only when `attention_type == "block_sparse"`.
num_random_blocks (`int`, *optional*, defaults to 3)
- Each query is going to attend these many number of random blocks. Useful only when `attention_type == "block_sparse"`.
+ Each query is going to attend these many number of random blocks. Useful only when `attention_type ==
+ "block_sparse"`.
scale_embeddings (`bool`, *optional*, defaults to `True`)
Whether to rescale embeddings with (hidden_size ** 0.5).
@@ -102,14 +102,13 @@ class BigBirdPegasusConfig(PretrainedConfig):
>>> from transformers import BigBirdPegasusModel, BigBirdPegasusConfig
- >>> # Initializing a BigBirdPegasus bigbird-pegasus-base style configuration
- >>> configuration = BigBirdPegasusConfig()
+ >>> # Initializing a BigBirdPegasus bigbird-pegasus-base style configuration >>> configuration =
+ BigBirdPegasusConfig()
- >>> # Initializing a model from the bigbird-pegasus-base style configuration
- >>> model = BigBirdPegasusModel(configuration)
+ >>> # Initializing a model from the bigbird-pegasus-base style configuration >>> model =
+ BigBirdPegasusModel(configuration)
- >>> # Accessing the model configuration
- >>> configuration = model.config
+ >>> # Accessing the model configuration >>> configuration = model.config
"""
model_type = "bigbird_pegasus"
keys_to_ignore_at_inference = ["past_key_values"]
diff --git a/src/transformers/models/bigbird_pegasus/modeling_bigbird_pegasus.py b/src/transformers/models/bigbird_pegasus/modeling_bigbird_pegasus.py
index 666fde2e5a..5eb5fda103 100755
--- a/src/transformers/models/bigbird_pegasus/modeling_bigbird_pegasus.py
+++ b/src/transformers/models/bigbird_pegasus/modeling_bigbird_pegasus.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" PyTorch BigBirdPegasus model. """
+""" PyTorch BigBirdPegasus model."""
import copy
@@ -1474,7 +1474,8 @@ class BigBirdPegasusDecoderLayer(nn.Module):
hidden_states (`torch.FloatTensor`): input to the layer of shape *(seq_len, batch, embed_dim)*
attention_mask (`torch.FloatTensor`): attention mask of size
*(batch, 1, tgt_len, src_len)* where padding elements are indicated by very large negative values.
- encoder_hidden_states (`torch.FloatTensor`): cross attention input to the layer of shape *(seq_len, batch, embed_dim)*
+ encoder_hidden_states (`torch.FloatTensor`):
+ cross attention input to the layer of shape *(seq_len, batch, embed_dim)*
encoder_attention_mask (`torch.FloatTensor`): encoder attention mask of size
*(batch, 1, tgt_len, src_len)* where padding elements are indicated by very large negative values.
layer_head_mask (`torch.FloatTensor`): mask for attention heads in a given layer of size
@@ -1603,13 +1604,12 @@ class BigBirdPegasusPreTrainedModel(PreTrainedModel):
BIGBIRD_PEGASUS_START_DOCSTRING = r"""
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings
- etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
Parameters:
config ([`BigBirdPegasusConfig`]):
@@ -1623,15 +1623,15 @@ BIGBIRD_PEGASUS_GENERATION_EXAMPLE = r"""
>>> from transformers import PegasusTokenizer, BigBirdPegasusForConditionalGeneration, BigBirdPegasusConfig
- >>> model = BigBirdPegasusForConditionalGeneration.from_pretrained('google/bigbird-pegasus-large-arxiv')
- >>> tokenizer = PegasusTokenizer.from_pretrained('google/bigbird-pegasus-large-arxiv')
+ >>> model = BigBirdPegasusForConditionalGeneration.from_pretrained('google/bigbird-pegasus-large-arxiv') >>>
+ tokenizer = PegasusTokenizer.from_pretrained('google/bigbird-pegasus-large-arxiv')
- >>> ARTICLE_TO_SUMMARIZE = "My friends are cool but they eat too many carbs."
- >>> inputs = tokenizer([ARTICLE_TO_SUMMARIZE], max_length=4096, return_tensors='pt', truncation=True)
+ >>> ARTICLE_TO_SUMMARIZE = "My friends are cool but they eat too many carbs." >>> inputs =
+ tokenizer([ARTICLE_TO_SUMMARIZE], max_length=4096, return_tensors='pt', truncation=True)
- >>> # Generate Summary
- >>> summary_ids = model.generate(inputs['input_ids'], num_beams=4, max_length=5, early_stopping=True)
- >>> print([tokenizer.decode(g, skip_special_tokens=True, clean_up_tokenization_spaces=False) for g in summary_ids])
+ >>> # Generate Summary >>> summary_ids = model.generate(inputs['input_ids'], num_beams=4, max_length=5,
+ early_stopping=True) >>> print([tokenizer.decode(g, skip_special_tokens=True,
+ clean_up_tokenization_spaces=False) for g in summary_ids])
"""
BIGBIRD_PEGASUS_INPUTS_DOCSTRING = r"""
@@ -1640,9 +1640,8 @@ BIGBIRD_PEGASUS_INPUTS_DOCSTRING = r"""
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
it.
- Indices can be obtained using [`PegasusTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`PegasusTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -1656,8 +1655,8 @@ BIGBIRD_PEGASUS_INPUTS_DOCSTRING = r"""
Provide for translation and summarization training. By default, the model will create this tensor by
shifting the `input_ids` to the right, following the paper.
decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
- Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will
- also be used by default.
+ Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
+ be used by default.
If you want to change padding behavior, you should read
[`modeling_bigbird_pegasus._prepare_decoder_inputs`] and modify to your needs. See diagram 1 in [the
@@ -1670,33 +1669,35 @@ BIGBIRD_PEGASUS_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*):
- Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*:
- `attentions`) `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`,
- *optional*) is a sequence of hidden-states at the output of the last layer of the encoder. Used in the
- cross-attention of the decoder.
+ Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`)
+ `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of
+ hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder.
past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors
- of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
- shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
+ Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
+ `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape
+ `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all ``decoder_input_ids``` of shape `(batch_size, sequence_length)`. inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ ``decoder_input_ids``` of shape `(batch_size, sequence_length)`. inputs_embeds (`torch.FloatTensor` of
+ shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids`
+ you can choose to directly pass an embedded representation. This is useful if you want more control over
+ how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup
+ matrix.
decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*):
Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded
- representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds`
- have to be input (see `past_key_values`). This is useful if you want more control over how to convert
+ representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be
+ input (see `past_key_values`). This is useful if you want more control over how to convert
`decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix.
- If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds`
- takes the value of `inputs_embeds`.
+ If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds` takes the value
+ of `inputs_embeds`.
use_cache (`bool`, *optional*):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`).
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`).
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -1713,9 +1714,8 @@ BIGBIRD_PEGASUS_STANDALONE_INPUTS_DOCSTRING = r"""
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
it.
- Indices can be obtained using [`ProphetNetTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`ProphetNetTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -1792,9 +1792,8 @@ class BigBirdPegasusEncoder(BigBirdPegasusPreTrainedModel):
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
provide it.
- Indices can be obtained using [`PegasusTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`]
- for details.
+ Indices can be obtained using [`PegasusTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -1806,9 +1805,9 @@ class BigBirdPegasusEncoder(BigBirdPegasusPreTrainedModel):
[What are attention masks?](../glossary#attention-mask)
inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded
- representation. This is useful if you want more control over how to convert `input_ids` indices
- into associated vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
+ This is useful if you want more control over how to convert `input_ids` indices into associated vectors
+ than the model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under
returned tensors for more detail.
@@ -2036,8 +2035,7 @@ class BigBirdPegasusEncoder(BigBirdPegasusPreTrainedModel):
class BigBirdPegasusDecoder(BigBirdPegasusPreTrainedModel):
"""
- Transformer decoder consisting of *config.decoder_layers* layers. Each layer is a
- [`BigBirdPegasusDecoderLayer`]
+ Transformer decoder consisting of *config.decoder_layers* layers. Each layer is a [`BigBirdPegasusDecoderLayer`]
Args:
config: BigBirdPegasusConfig
@@ -2114,9 +2112,8 @@ class BigBirdPegasusDecoder(BigBirdPegasusPreTrainedModel):
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
provide it.
- Indices can be obtained using [`PegasusTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`]
- for details.
+ Indices can be obtained using [`PegasusTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -2151,19 +2148,20 @@ class BigBirdPegasusDecoder(BigBirdPegasusPreTrainedModel):
- 0 indicates the head is **masked**.
past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2
- tensors of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional
- tensors of shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
+ Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of
+ shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
+ shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
Contains pre-computed hidden-states (key and values in the self-attention blocks and in the
- cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential
- decoding.
+ cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
- If `past_key_values` are used, the user can optionally input only the last
- `decoder_input_ids` (those that don't have their past key value states given to this model) of
- shape `(batch_size, 1)` instead of all ``decoder_input_ids``` of shape `(batch_size,
- sequence_length)`. inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices
- into associated vectors than the model's internal embedding lookup matrix.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those
+ that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of
+ all ``decoder_input_ids``` of shape `(batch_size, sequence_length)`. inputs_embeds (`torch.FloatTensor`
+ of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing
+ `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more
+ control over how to convert `input_ids` indices into associated vectors than the model's internal
+ embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under
returned tensors for more detail.
@@ -2504,7 +2502,8 @@ class BigBirdPegasusForConditionalGeneration(BigBirdPegasusPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
+ Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
+ config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
(masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
Returns:
@@ -2647,7 +2646,8 @@ class BigBirdPegasusForSequenceClassification(BigBirdPegasusPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
if labels is not None:
@@ -2916,9 +2916,8 @@ class BigBirdPegasusForCausalLM(BigBirdPegasusPreTrainedModel):
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
provide it.
- Indices can be obtained using [`PegasusTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`]
- for details.
+ Indices can be obtained using [`PegasusTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -2947,25 +2946,24 @@ class BigBirdPegasusForCausalLM(BigBirdPegasusPreTrainedModel):
- 0 indicates the head is **masked**.
past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2
- tensors of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional
- tensors of shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. The two
- additional tensors are only required when the model is used as a decoder in a Sequence to Sequence
- model.
+ Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of
+ shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
+ shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. The two additional
+ tensors are only required when the model is used as a decoder in a Sequence to Sequence model.
Contains pre-computed hidden-states (key and values in the self-attention blocks and in the
- cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential
- decoding.
+ cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those
+ that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of
+ all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are
- ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
+ Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
+ config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
+ (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
use_cache (`bool`, *optional*):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`).
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
+ (see `past_key_values`).
- 1 for tokens that are **not masked**,
- 0 for tokens that are **masked**.
diff --git a/src/transformers/models/blenderbot/configuration_blenderbot.py b/src/transformers/models/blenderbot/configuration_blenderbot.py
index 5dccf86d9c..fac9250f64 100644
--- a/src/transformers/models/blenderbot/configuration_blenderbot.py
+++ b/src/transformers/models/blenderbot/configuration_blenderbot.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" Blenderbot model configuration """
+""" Blenderbot model configuration"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging
@@ -28,20 +28,19 @@ BLENDERBOT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class BlenderbotConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`BlenderbotModel`]. It is used
- to instantiate an Blenderbot model according to the specified arguments, defining the model architecture.
- Instantiating a configuration with the defaults will yield a similar configuration to that of the Blenderbot
+ This is the configuration class to store the configuration of a [`BlenderbotModel`]. It is used to instantiate an
+ Blenderbot model according to the specified arguments, defining the model architecture. Instantiating a
+ configuration with the defaults will yield a similar configuration to that of the Blenderbot
[facebook/blenderbot-3B](https://huggingface.co/facebook/blenderbot-3B) architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
vocab_size (`int`, *optional*, defaults to 50265):
Vocabulary size of the Blenderbot model. Defines the number of different tokens that can be represented by
- the `inputs_ids` passed when calling [`BlenderbotModel`] or
- [`TFBlenderbotModel`].
+ the `inputs_ids` passed when calling [`BlenderbotModel`] or [`TFBlenderbotModel`].
d_model (`int`, *optional*, defaults to 1024):
Dimensionality of the layers and the pooler layer.
encoder_layers (`int`, *optional*, defaults to 12):
@@ -57,8 +56,8 @@ class BlenderbotConfig(PretrainedConfig):
encoder_ffn_dim (`int`, *optional*, defaults to 4096):
Dimensionality of the "intermediate" (often named feed-forward) layer in decoder.
activation_function (`str` or `function`, *optional*, defaults to `"gelu"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"silu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"silu"` and `"gelu_new"` are supported.
dropout (`float`, *optional*, defaults to 0.1):
The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
attention_dropout (`float`, *optional*, defaults to 0.0):
@@ -73,11 +72,11 @@ class BlenderbotConfig(PretrainedConfig):
init_std (`float`, *optional*, defaults to 0.02):
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
encoder_layerdrop: (`float`, *optional*, defaults to 0.0):
- The LayerDrop probability for the encoder. See the [LayerDrop paper](see
- https://arxiv.org/abs/1909.11556) for more details.
+ The LayerDrop probability for the encoder. See the [LayerDrop paper](see https://arxiv.org/abs/1909.11556)
+ for more details.
decoder_layerdrop: (`float`, *optional*, defaults to 0.0):
- The LayerDrop probability for the decoder. See the [LayerDrop paper](see
- https://arxiv.org/abs/1909.11556) for more details.
+ The LayerDrop probability for the decoder. See the [LayerDrop paper](see https://arxiv.org/abs/1909.11556)
+ for more details.
scale_embedding (`bool`, *optional*, defaults to `False`):
Scale embeddings by diving by sqrt(d_model).
use_cache (`bool`, *optional*, defaults to `True`):
diff --git a/src/transformers/models/blenderbot/modeling_blenderbot.py b/src/transformers/models/blenderbot/modeling_blenderbot.py
index fc9597f76e..b2e7f0b5a8 100755
--- a/src/transformers/models/blenderbot/modeling_blenderbot.py
+++ b/src/transformers/models/blenderbot/modeling_blenderbot.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" PyTorch Blenderbot model. """
+""" PyTorch Blenderbot model."""
import copy
@@ -384,7 +384,8 @@ class BlenderbotDecoderLayer(nn.Module):
hidden_states (`torch.FloatTensor`): input to the layer of shape *(seq_len, batch, embed_dim)*
attention_mask (`torch.FloatTensor`): attention mask of size
*(batch, 1, tgt_len, src_len)* where padding elements are indicated by very large negative values.
- encoder_hidden_states (`torch.FloatTensor`): cross attention input to the layer of shape *(seq_len, batch, embed_dim)*
+ encoder_hidden_states (`torch.FloatTensor`):
+ cross attention input to the layer of shape *(seq_len, batch, embed_dim)*
encoder_attention_mask (`torch.FloatTensor`): encoder attention mask of size
*(batch, 1, tgt_len, src_len)* where padding elements are indicated by very large negative values.
layer_head_mask (`torch.FloatTensor`): mask for attention heads in a given layer of size
@@ -489,13 +490,13 @@ class BlenderbotPreTrainedModel(PreTrainedModel):
BLENDERBOT_START_DOCSTRING = r"""
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
Parameters:
config ([`BlenderbotConfig`]):
@@ -507,26 +508,18 @@ BLENDERBOT_START_DOCSTRING = r"""
BLENDERBOT_GENERATION_EXAMPLE = r"""
Conversation example::
- >>> from transformers import BlenderbotTokenizer, BlenderbotForConditionalGeneration
- >>> mname = 'facebook/blenderbot-400M-distill'
- >>> model = BlenderbotForConditionalGeneration.from_pretrained(mname)
- >>> tokenizer = BlenderbotTokenizer.from_pretrained(mname)
- >>> UTTERANCE = "My friends are cool but they eat too many carbs."
- >>> print("Human: ", UTTERANCE)
- >>> inputs = tokenizer([UTTERANCE], return_tensors='pt')
- >>> reply_ids = model.generate(**inputs)
- >>> print("Bot: ", tokenizer.batch_decode(reply_ids, skip_special_tokens=True)[0])
+ >>> from transformers import BlenderbotTokenizer, BlenderbotForConditionalGeneration >>> mname =
+ 'facebook/blenderbot-400M-distill' >>> model = BlenderbotForConditionalGeneration.from_pretrained(mname) >>>
+ tokenizer = BlenderbotTokenizer.from_pretrained(mname) >>> UTTERANCE = "My friends are cool but they eat too
+ many carbs." >>> print("Human: ", UTTERANCE) >>> inputs = tokenizer([UTTERANCE], return_tensors='pt') >>>
+ reply_ids = model.generate(**inputs) >>> print("Bot: ", tokenizer.batch_decode(reply_ids,
+ skip_special_tokens=True)[0])
- >>> REPLY = "I'm not sure"
- >>> print("Human: ", REPLY)
- >>> NEXT_UTTERANCE = (
- ... "My friends are cool but they eat too many carbs. That's unfortunate. "
- ... "Are they trying to lose weight or are they just trying to be healthier? "
- ... " I'm not sure."
- ... )
- >>> inputs = tokenizer([NEXT_UTTERANCE], return_tensors='pt')
- >>> next_reply_ids = model.generate(**inputs)
- >>> print("Bot: ", tokenizer.batch_decode(next_reply_ids, skip_special_tokens=True)[0])
+ >>> REPLY = "I'm not sure" >>> print("Human: ", REPLY) >>> NEXT_UTTERANCE = ( ... "My friends are cool but they
+ eat too many carbs. That's unfortunate. " ... "Are they trying to lose weight or are they just trying to
+ be healthier? " ... " I'm not sure." ... ) >>> inputs = tokenizer([NEXT_UTTERANCE], return_tensors='pt')
+ >>> next_reply_ids = model.generate(**inputs) >>> print("Bot: ", tokenizer.batch_decode(next_reply_ids,
+ skip_special_tokens=True)[0])
"""
BLENDERBOT_INPUTS_DOCSTRING = r"""
@@ -535,9 +528,8 @@ BLENDERBOT_INPUTS_DOCSTRING = r"""
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
it.
- Indices can be obtained using [`BlenderbotTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`BlenderbotTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -550,9 +542,8 @@ BLENDERBOT_INPUTS_DOCSTRING = r"""
decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
Indices of decoder input sequence tokens in the vocabulary.
- Indices can be obtained using [`BlenderbotTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`BlenderbotTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are decoder input IDs?](../glossary#decoder-input-ids)
@@ -560,8 +551,8 @@ BLENDERBOT_INPUTS_DOCSTRING = r"""
`past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
`past_key_values`).
decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
- Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will
- also be used by default.
+ Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
+ be used by default.
head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*):
Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`:
@@ -575,39 +566,42 @@ BLENDERBOT_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*):
- Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, 1]`:
+ Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0,
+ 1]`:
- 1 indicates the head is **not masked**,
- 0 indicates the head is **masked**.
encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*):
- Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*:
- `attentions`) `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`,
- *optional*) is a sequence of hidden-states at the output of the last layer of the encoder. Used in the
- cross-attention of the decoder.
+ Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`)
+ `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of
+ hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder.
past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors
- of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
- shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
+ Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
+ `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape
+ `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all ``decoder_input_ids``` of shape `(batch_size, sequence_length)`. inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ ``decoder_input_ids``` of shape `(batch_size, sequence_length)`. inputs_embeds (`torch.FloatTensor` of
+ shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids`
+ you can choose to directly pass an embedded representation. This is useful if you want more control over
+ how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup
+ matrix.
decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*):
Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded
- representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds`
- have to be input (see `past_key_values`). This is useful if you want more control over how to convert
+ representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be
+ input (see `past_key_values`). This is useful if you want more control over how to convert
`decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix.
- If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds`
- takes the value of `inputs_embeds`.
+ If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds` takes the value
+ of `inputs_embeds`.
use_cache (`bool`, *optional*):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`).
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`).
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -672,9 +666,8 @@ class BlenderbotEncoder(BlenderbotPreTrainedModel):
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
provide it.
- Indices can be obtained using [`BlenderbotTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`]
- for details.
+ Indices can be obtained using [`BlenderbotTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -691,9 +684,9 @@ class BlenderbotEncoder(BlenderbotPreTrainedModel):
- 0 indicates the head is **masked**.
inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded
- representation. This is useful if you want more control over how to convert `input_ids` indices
- into associated vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
+ This is useful if you want more control over how to convert `input_ids` indices into associated vectors
+ than the model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under
returned tensors for more detail.
@@ -868,9 +861,8 @@ class BlenderbotDecoder(BlenderbotPreTrainedModel):
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
provide it.
- Indices can be obtained using [`BlenderbotTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`]
- for details.
+ Indices can be obtained using [`BlenderbotTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -892,7 +884,8 @@ class BlenderbotDecoder(BlenderbotPreTrainedModel):
[What are attention masks?](../glossary#attention-mask)
head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*):
- Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`:
+ Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0,
+ 1]`:
- 1 indicates the head is **not masked**,
- 0 indicates the head is **masked**.
@@ -905,19 +898,20 @@ class BlenderbotDecoder(BlenderbotPreTrainedModel):
- 0 indicates the head is **masked**.
past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2
- tensors of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional
- tensors of shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
+ Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of
+ shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
+ shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
Contains pre-computed hidden-states (key and values in the self-attention blocks and in the
- cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential
- decoding.
+ cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
- If `past_key_values` are used, the user can optionally input only the last
- `decoder_input_ids` (those that don't have their past key value states given to this model) of
- shape `(batch_size, 1)` instead of all ``decoder_input_ids``` of shape `(batch_size,
- sequence_length)`. inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices
- into associated vectors than the model's internal embedding lookup matrix.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those
+ that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of
+ all ``decoder_input_ids``` of shape `(batch_size, sequence_length)`. inputs_embeds (`torch.FloatTensor`
+ of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing
+ `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more
+ control over how to convert `input_ids` indices into associated vectors than the model's internal
+ embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under
returned tensors for more detail.
@@ -1282,7 +1276,8 @@ class BlenderbotForConditionalGeneration(BlenderbotPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
+ Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
+ config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
(masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
Returns:
@@ -1444,9 +1439,8 @@ class BlenderbotForCausalLM(BlenderbotPreTrainedModel):
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
provide it.
- Indices can be obtained using [`BlenderbotTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`]
- for details.
+ Indices can be obtained using [`BlenderbotTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -1475,25 +1469,24 @@ class BlenderbotForCausalLM(BlenderbotPreTrainedModel):
- 0 indicates the head is **masked**.
past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2
- tensors of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional
- tensors of shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. The two
- additional tensors are only required when the model is used as a decoder in a Sequence to Sequence
- model.
+ Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of
+ shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
+ shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. The two additional
+ tensors are only required when the model is used as a decoder in a Sequence to Sequence model.
Contains pre-computed hidden-states (key and values in the self-attention blocks and in the
- cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential
- decoding.
+ cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those
+ that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of
+ all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are
- ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
+ Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
+ config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
+ (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
use_cache (`bool`, *optional*):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`).
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
+ (see `past_key_values`).
- 1 for tokens that are **not masked**,
- 0 for tokens that are **masked**.
diff --git a/src/transformers/models/blenderbot/modeling_flax_blenderbot.py b/src/transformers/models/blenderbot/modeling_flax_blenderbot.py
index f3dcf35f64..cd41a0c85f 100644
--- a/src/transformers/models/blenderbot/modeling_flax_blenderbot.py
+++ b/src/transformers/models/blenderbot/modeling_flax_blenderbot.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" Flax Blenderbot model. """
+""" Flax Blenderbot model."""
import math
import random
@@ -57,12 +57,13 @@ _CHECKPOINT_FOR_DOC = "facebook/blenderbot-400M-distill"
BLENDERBOT_START_DOCSTRING = r"""
- This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading or saving, resizing the input
- embeddings, pruning heads etc.)
+ This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a Flax Linen [flax.nn.Module](https://flax.readthedocs.io/en/latest/_autosummary/flax.nn.module.html) subclass. Use it as a regular Flax
- Module and refer to the Flax documentation for all matter related to general usage and behavior.
+ This model is also a Flax Linen
+ [flax.nn.Module](https://flax.readthedocs.io/en/latest/_autosummary/flax.nn.module.html) subclass. Use it as a
+ regular Flax Module and refer to the Flax documentation for all matter related to general usage and behavior.
Finally, this model supports inherent JAX features such as:
@@ -74,8 +75,7 @@ BLENDERBOT_START_DOCSTRING = r"""
Parameters:
config ([`BlenderbotConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the
- model weights.
+ configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the model weights.
"""
BLENDERBOT_INPUTS_DOCSTRING = r"""
@@ -84,9 +84,8 @@ BLENDERBOT_INPUTS_DOCSTRING = r"""
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
it.
- Indices can be obtained using [`BlenderbotTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`BlenderbotTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`jnp.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
@@ -99,22 +98,23 @@ BLENDERBOT_INPUTS_DOCSTRING = r"""
decoder_input_ids (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`, *optional*):
Indices of decoder input sequence tokens in the vocabulary.
- Indices can be obtained using [`BlenderbotTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`BlenderbotTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are decoder input IDs?](../glossary#decoder-input-ids)
For translation and summarization training, `decoder_input_ids` should be provided. If no
- `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to
- the right for denoising pre-training following the paper.
+ `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right
+ for denoising pre-training following the paper.
decoder_attention_mask (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`, *optional*):
- Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will
- also be used by default.
+ Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
+ be used by default.
- If you want to change padding behavior, you should modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy.
+ If you want to change padding behavior, you should modify to your needs. See diagram 1 in [the
+ paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy.
position_ids (`numpy.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
decoder_position_ids (`numpy.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the
range `[0, config.max_position_embeddings - 1]`.
@@ -135,9 +135,8 @@ BLENDERBOT_ENCODE_INPUTS_DOCSTRING = r"""
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
it.
- Indices can be obtained using [`BlenderbotTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`BlenderbotTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`jnp.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
@@ -148,7 +147,8 @@ BLENDERBOT_ENCODE_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
position_ids (`numpy.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -164,20 +164,18 @@ BLENDERBOT_DECODE_INPUTS_DOCSTRING = r"""
decoder_input_ids (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`):
Indices of decoder input sequence tokens in the vocabulary.
- Indices can be obtained using [`BlenderbotTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`BlenderbotTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are decoder input IDs?](../glossary#decoder-input-ids)
For translation and summarization training, `decoder_input_ids` should be provided. If no
- `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to
- the right for denoising pre-training following the paper.
+ `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right
+ for denoising pre-training following the paper.
encoder_outputs (`tuple(tuple(jnp.ndarray)`):
- Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*:
- `attentions`) `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`,
- *optional*) is a sequence of hidden-states at the output of the last layer of the encoder. Used in the
- cross-attention of the decoder.
+ Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`)
+ `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of
+ hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder.
encoder_attention_mask (`jnp.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
@@ -186,10 +184,11 @@ BLENDERBOT_DECODE_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
decoder_attention_mask (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`, *optional*):
- Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will
- also be used by default.
+ Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
+ be used by default.
- If you want to change padding behavior, you should modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy.
+ If you want to change padding behavior, you should modify to your needs. See diagram 1 in [the
+ paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy.
decoder_position_ids (`numpy.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the
range `[0, config.max_position_embeddings - 1]`.
@@ -929,9 +928,10 @@ class FlaxBlenderbotPreTrainedModel(FlaxPreTrainedModel):
maximum possible length for auto-regressive decoding. Defines the sequence length of the initialized
cache.
encoder_outputs (`Union[FlaxBaseModelOutput, tuple(tuple(jnp.ndarray)]`):
- `encoder_outputs` consists of (`last_hidden_state`, *optional*: `hidden_states`,
- *optional*: `attentions`). `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of hidden-states at the output of the last layer of the
- encoder. Used in the cross-attention of the decoder.
+ `encoder_outputs` consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*:
+ `attentions`). `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*)
+ is a sequence of hidden-states at the output of the last layer of the encoder. Used in the
+ cross-attention of the decoder.
"""
# init input variables to retrieve cache
decoder_input_ids = jnp.ones((batch_size, max_length), dtype="i4")
@@ -1474,15 +1474,15 @@ FLAX_BLENDERBOT_CONDITIONAL_GENERATION_DOCSTRING = r"""
>>> from transformers import BlenderbotTokenizer, FlaxBlenderbotForConditionalGeneration, BlenderbotConfig
- >>> model = FlaxBlenderbotForConditionalGeneration.from_pretrained('facebook/blenderbot-400M-distill')
- >>> tokenizer = BlenderbotTokenizer.from_pretrained('facebook/blenderbot-400M-distill')
+ >>> model = FlaxBlenderbotForConditionalGeneration.from_pretrained('facebook/blenderbot-400M-distill') >>>
+ tokenizer = BlenderbotTokenizer.from_pretrained('facebook/blenderbot-400M-distill')
- >>> UTTERANCE = "My friends are cool but they eat too many carbs."
- >>> inputs = tokenizer([UTTERANCE], max_length=1024, return_tensors='np')
+ >>> UTTERANCE = "My friends are cool but they eat too many carbs." >>> inputs = tokenizer([UTTERANCE],
+ max_length=1024, return_tensors='np')
- >>> # Generate Reply
- >>> reply_ids = model.generate(inputs['input_ids'], num_beams=4, max_length=5, early_stopping=True).sequences
- >>> print([tokenizer.decode(g, skip_special_tokens=True, clean_up_tokenization_spaces=False) for g in reply_ids])
+ >>> # Generate Reply >>> reply_ids = model.generate(inputs['input_ids'], num_beams=4, max_length=5,
+ early_stopping=True).sequences >>> print([tokenizer.decode(g, skip_special_tokens=True,
+ clean_up_tokenization_spaces=False) for g in reply_ids])
"""
overwrite_call_docstring(
diff --git a/src/transformers/models/blenderbot/modeling_tf_blenderbot.py b/src/transformers/models/blenderbot/modeling_tf_blenderbot.py
index 24ae231c0c..956afd85c7 100644
--- a/src/transformers/models/blenderbot/modeling_tf_blenderbot.py
+++ b/src/transformers/models/blenderbot/modeling_tf_blenderbot.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" TF 2.0 Blenderbot model. """
+""" TF 2.0 Blenderbot model."""
import os
@@ -380,7 +380,8 @@ class TFBlenderbotDecoderLayer(tf.keras.layers.Layer):
hidden_states (`tf.Tensor`): input to the layer of shape *(seq_len, batch, embed_dim)*
attention_mask (`tf.Tensor`): attention mask of size
*(batch, 1, tgt_len, src_len)* where padding elements are indicated by very large negative values.
- encoder_hidden_states (`tf.Tensor`): cross attention input to the layer of shape *(seq_len, batch, embed_dim)*
+ encoder_hidden_states (`tf.Tensor`):
+ cross attention input to the layer of shape *(seq_len, batch, embed_dim)*
encoder_attention_mask (`tf.Tensor`): encoder attention mask of size
*(batch, 1, tgt_len, src_len)* where padding elements are indicated by very large negative values.
layer_head_mask (`tf.Tensor`): mask for attention heads in a given layer of size
@@ -478,13 +479,13 @@ class TFBlenderbotPreTrainedModel(TFPreTrainedModel):
BLENDERBOT_START_DOCSTRING = r"""
- This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading or saving, resizing the input
- embeddings, pruning heads etc.)
+ This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use
- it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage
- and behavior.
+ This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it
+ as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and
+ behavior.
@@ -493,11 +494,11 @@ BLENDERBOT_START_DOCSTRING = r"""
- having all inputs as keyword arguments (like PyTorch models), or
- having all inputs as a list, tuple or dict in the first positional arguments.
- This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all
- the tensors in the first argument of the model call function: `model(inputs)`.
+ This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all the
+ tensors in the first argument of the model call function: `model(inputs)`.
- If you choose this second option, there are three possibilities you can use to gather all the input Tensors in
- the first positional argument :
+ If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the
+ first positional argument :
- a single Tensor with `input_ids` only and nothing else: `model(input_ids)`
- a list of varying length with one or several input Tensors IN THE ORDER given in the docstring:
@@ -510,33 +511,24 @@ BLENDERBOT_START_DOCSTRING = r"""
Args:
config ([`BlenderbotConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~TFPreTrainedModel.from_pretrained`] method to load the
- model weights.
+ configuration. Check out the [`~TFPreTrainedModel.from_pretrained`] method to load the model weights.
"""
BLENDERBOT_GENERATION_EXAMPLE = r"""
Conversation example::
- >>> from transformers import BlenderbotTokenizer, TFBlenderbotForConditionalGeneration
- >>> mname = 'facebook/blenderbot-400M-distill'
- >>> model = TFBlenderbotForConditionalGeneration.from_pretrained(mname)
- >>> tokenizer = BlenderbotTokenizer.from_pretrained(mname)
- >>> UTTERANCE = "My friends are cool but they eat too many carbs."
- >>> print("Human: ", UTTERANCE)
- >>> inputs = tokenizer([UTTERANCE], return_tensors='tf')
- >>> reply_ids = model.generate(**inputs)
- >>> print("Bot: ", tokenizer.batch_decode(reply_ids, skip_special_tokens=True)[0])
+ >>> from transformers import BlenderbotTokenizer, TFBlenderbotForConditionalGeneration >>> mname =
+ 'facebook/blenderbot-400M-distill' >>> model = TFBlenderbotForConditionalGeneration.from_pretrained(mname) >>>
+ tokenizer = BlenderbotTokenizer.from_pretrained(mname) >>> UTTERANCE = "My friends are cool but they eat too
+ many carbs." >>> print("Human: ", UTTERANCE) >>> inputs = tokenizer([UTTERANCE], return_tensors='tf') >>>
+ reply_ids = model.generate(**inputs) >>> print("Bot: ", tokenizer.batch_decode(reply_ids,
+ skip_special_tokens=True)[0])
- >>> REPLY = "I'm not sure"
- >>> print("Human: ", REPLY)
- >>> NEXT_UTTERANCE = (
- ... "My friends are cool but they eat too many carbs. That's unfortunate. "
- ... "Are they trying to lose weight or are they just trying to be healthier? "
- ... " I'm not sure."
- ... )
- >>> inputs = tokenizer([NEXT_UTTERANCE], return_tensors='tf')
- >>> next_reply_ids = model.generate(**inputs)
- >>> print("Bot: ", tokenizer.batch_decode(next_reply_ids, skip_special_tokens=True)[0])
+ >>> REPLY = "I'm not sure" >>> print("Human: ", REPLY) >>> NEXT_UTTERANCE = ( ... "My friends are cool but they
+ eat too many carbs. That's unfortunate. " ... "Are they trying to lose weight or are they just trying to
+ be healthier? " ... " I'm not sure." ... ) >>> inputs = tokenizer([NEXT_UTTERANCE], return_tensors='tf')
+ >>> next_reply_ids = model.generate(**inputs) >>> print("Bot: ", tokenizer.batch_decode(next_reply_ids,
+ skip_special_tokens=True)[0])
"""
BLENDERBOT_INPUTS_DOCSTRING = r"""
@@ -544,9 +536,8 @@ BLENDERBOT_INPUTS_DOCSTRING = r"""
input_ids (`tf.Tensor` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`BlenderbotTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`BlenderbotTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`tf.Tensor` of shape `({0})`, *optional*):
@@ -559,9 +550,8 @@ BLENDERBOT_INPUTS_DOCSTRING = r"""
decoder_input_ids (`tf.Tensor` of shape `(batch_size, target_sequence_length)`, *optional*):
Indices of decoder input sequence tokens in the vocabulary.
- Indices can be obtained using [`BlenderbotTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`BlenderbotTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are decoder input IDs?](../glossary#decoder-input-ids)
@@ -593,12 +583,12 @@ BLENDERBOT_INPUTS_DOCSTRING = r"""
of shape `(batch_size, sequence_length, hidden_size)` is a sequence of
past_key_values (`Tuple[Tuple[tf.Tensor]]` of length `config.n_layers`)
contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ `decoder_input_ids` of shape `(batch_size, sequence_length)`.
use_cache (`bool`, *optional*, defaults to `True`):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`). Set to `False` during training, `True` during generation
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`). Set to `False` during training, `True` during generation
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the
@@ -608,8 +598,8 @@ BLENDERBOT_INPUTS_DOCSTRING = r"""
more detail. This argument can be used only in eager mode, in graph mode the value in the config will be
used instead.
return_dict (`bool`, *optional*):
- Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This
- argument can be used in eager mode, in graph mode the value will always be set to True.
+ Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This argument can be used
+ in eager mode, in graph mode the value will always be set to True.
training (`bool`, *optional*, defaults to `False`):
Whether or not to use the model in training mode (some modules like dropout modules have different
behaviors between training and evaluation).
@@ -669,9 +659,8 @@ class TFBlenderbotEncoder(tf.keras.layers.Layer):
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
provide it.
- Indices can be obtained using [`BlenderbotTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`]
- for details.
+ Indices can be obtained using [`BlenderbotTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -681,15 +670,16 @@ class TFBlenderbotEncoder(tf.keras.layers.Layer):
- 0 for tokens that are **masked**.
[What are attention masks?](../glossary#attention-mask)
- head_mask (`tf.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, `optional): Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`:
+ head_mask (`tf.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, `optional):
+ Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`:
- 1 indicates the head is **not masked**,
- 0 indicates the head is **masked**.
inputs_embeds (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded
- representation. This is useful if you want more control over how to convert `input_ids` indices
- into associated vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
+ This is useful if you want more control over how to convert `input_ids` indices into associated vectors
+ than the model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under
returned tensors for more detail. This argument can be used only in eager mode, in graph mode the value
@@ -699,8 +689,8 @@ class TFBlenderbotEncoder(tf.keras.layers.Layer):
for more detail. This argument can be used only in eager mode, in graph mode the value in the config
will be used instead.
return_dict (`bool`, *optional*):
- Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This
- argument can be used in eager mode, in graph mode the value will always be set to True.
+ Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This argument can be
+ used in eager mode, in graph mode the value will always be set to True.
training (`bool`, *optional*, defaults to `False`):
Whether or not to use the model in training mode (some modules like dropout modules have different
behaviors between training and evaluation).
@@ -843,9 +833,8 @@ class TFBlenderbotDecoder(tf.keras.layers.Layer):
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
provide it.
- Indices can be obtained using [`BlenderbotTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`]
- for details.
+ Indices can be obtained using [`BlenderbotTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -882,11 +871,13 @@ class TFBlenderbotDecoder(tf.keras.layers.Layer):
Contains precomputed key and value hidden-states of the attention blocks. Can be used to speed up
decoding.
- If `past_key_values` are used, the user can optionally input only the last
- `decoder_input_ids` (those that don't have their past key value states given to this model) of
- shape `(batch_size, 1)` instead of all ``decoder_input_ids``` of shape `(batch_size,
- sequence_length)`. inputs_embeds (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices
- into associated vectors than the model's internal embedding lookup matrix.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those
+ that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of
+ all ``decoder_input_ids``` of shape `(batch_size, sequence_length)`. inputs_embeds (`tf.Tensor` of
+ shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing
+ `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more
+ control over how to convert `input_ids` indices into associated vectors than the model's internal
+ embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under
returned tensors for more detail. This argument can be used only in eager mode, in graph mode the value
@@ -896,8 +887,8 @@ class TFBlenderbotDecoder(tf.keras.layers.Layer):
for more detail. This argument can be used only in eager mode, in graph mode the value in the config
will be used instead.
return_dict (`bool`, *optional*):
- Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This
- argument can be used in eager mode, in graph mode the value will always be set to True.
+ Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This argument can be
+ used in eager mode, in graph mode the value will always be set to True.
training (`bool`, *optional*, defaults to `False`):
Whether or not to use the model in training mode (some modules like dropout modules have different
behaviors between training and evaluation).
@@ -1369,7 +1360,8 @@ class TFBlenderbotForConditionalGeneration(TFBlenderbotPreTrainedModel, TFCausal
):
r"""
labels (`tf.tensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
+ Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
+ config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
(masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
Returns:
diff --git a/src/transformers/models/blenderbot/tokenization_blenderbot.py b/src/transformers/models/blenderbot/tokenization_blenderbot.py
index 966b1294db..8fabbbf6f2 100644
--- a/src/transformers/models/blenderbot/tokenization_blenderbot.py
+++ b/src/transformers/models/blenderbot/tokenization_blenderbot.py
@@ -47,12 +47,10 @@ class BlenderbotTokenizer(RobertaTokenizer):
r"""
Construct a Blenderbot tokenizer.
- [`Blenderbot`] is nearly identical to [`RobertaTokenizer`] and runs
- end-to-end tokenization: punctuation splitting and wordpiece. The only difference is that it doesn't add BOS token
- to the beginning of sequences.
+ [`Blenderbot`] is nearly identical to [`RobertaTokenizer`] and runs end-to-end tokenization: punctuation splitting
+ and wordpiece. The only difference is that it doesn't add BOS token to the beginning of sequences.
- Refer to superclass [`RobertaTokenizer`] for usage examples and documentation concerning
- parameters.
+ Refer to superclass [`RobertaTokenizer`] for usage examples and documentation concerning parameters.
"""
vocab_files_names = VOCAB_FILES_NAMES
pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
diff --git a/src/transformers/models/blenderbot/tokenization_blenderbot_fast.py b/src/transformers/models/blenderbot/tokenization_blenderbot_fast.py
index f04ce1b369..258068b9c3 100644
--- a/src/transformers/models/blenderbot/tokenization_blenderbot_fast.py
+++ b/src/transformers/models/blenderbot/tokenization_blenderbot_fast.py
@@ -48,12 +48,10 @@ class BlenderbotTokenizerFast(RobertaTokenizerFast):
r"""
Construct a "fast" Blenderbot tokenizer (backed by HuggingFace's *tokenizers* library).
- [`BlenderbotFast`] is nearly identical to [`RobertaTokenizerFast`] and runs
- end-to-end tokenization: punctuation splitting and wordpiece. The only difference is that it doesn't add BOS token
- to the beginning of sequences.
+ [`BlenderbotFast`] is nearly identical to [`RobertaTokenizerFast`] and runs end-to-end tokenization: punctuation
+ splitting and wordpiece. The only difference is that it doesn't add BOS token to the beginning of sequences.
- Refer to superclass [`RobertaTokenizerFast`] for usage examples and documentation concerning
- parameters.
+ Refer to superclass [`RobertaTokenizerFast`] for usage examples and documentation concerning parameters.
"""
vocab_files_names = VOCAB_FILES_NAMES
pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
diff --git a/src/transformers/models/blenderbot_small/configuration_blenderbot_small.py b/src/transformers/models/blenderbot_small/configuration_blenderbot_small.py
index 2490cb0207..6bcf40e9f0 100644
--- a/src/transformers/models/blenderbot_small/configuration_blenderbot_small.py
+++ b/src/transformers/models/blenderbot_small/configuration_blenderbot_small.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" BlenderbotSmall model configuration """
+""" BlenderbotSmall model configuration"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging
@@ -28,20 +28,19 @@ BLENDERBOT_SMALL_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class BlenderbotSmallConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`BlenderbotSmallModel`]. It is
- used to instantiate an BlenderbotSmall model according to the specified arguments, defining the model architecture.
- Instantiating a configuration with the defaults will yield a similar configuration to that of the BlenderbotSmall
+ This is the configuration class to store the configuration of a [`BlenderbotSmallModel`]. It is used to instantiate
+ an BlenderbotSmall model according to the specified arguments, defining the model architecture. Instantiating a
+ configuration with the defaults will yield a similar configuration to that of the BlenderbotSmall
[facebook/blenderbot_small-90M](https://huggingface.co/facebook/blenderbot_small-90M) architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
vocab_size (`int`, *optional*, defaults to 50265):
Vocabulary size of the BlenderbotSmall model. Defines the number of different tokens that can be
- represented by the `inputs_ids` passed when calling [`BlenderbotSmallModel`] or
- [`TFBlenderbotSmallModel`].
+ represented by the `inputs_ids` passed when calling [`BlenderbotSmallModel`] or [`TFBlenderbotSmallModel`].
d_model (`int`, *optional*, defaults to 512):
Dimensionality of the layers and the pooler layer.
encoder_layers (`int`, *optional*, defaults to 8):
@@ -57,8 +56,8 @@ class BlenderbotSmallConfig(PretrainedConfig):
encoder_ffn_dim (`int`, *optional*, defaults to 2048):
Dimensionality of the "intermediate" (often named feed-forward) layer in decoder.
activation_function (`str` or `function`, *optional*, defaults to `"gelu"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"silu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"silu"` and `"gelu_new"` are supported.
dropout (`float`, *optional*, defaults to 0.1):
The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
attention_dropout (`float`, *optional*, defaults to 0.0):
@@ -73,11 +72,11 @@ class BlenderbotSmallConfig(PretrainedConfig):
init_std (`float`, *optional*, defaults to 0.02):
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
encoder_layerdrop: (`float`, *optional*, defaults to 0.0):
- The LayerDrop probability for the encoder. See the [LayerDrop paper](see
- https://arxiv.org/abs/1909.11556) for more details.
+ The LayerDrop probability for the encoder. See the [LayerDrop paper](see https://arxiv.org/abs/1909.11556)
+ for more details.
decoder_layerdrop: (`float`, *optional*, defaults to 0.0):
- The LayerDrop probability for the decoder. See the [LayerDrop paper](see
- https://arxiv.org/abs/1909.11556) for more details.
+ The LayerDrop probability for the decoder. See the [LayerDrop paper](see https://arxiv.org/abs/1909.11556)
+ for more details.
scale_embedding (`bool`, *optional*, defaults to `False`):
Scale embeddings by diving by sqrt(d_model).
use_cache (`bool`, *optional*, defaults to `True`):
diff --git a/src/transformers/models/blenderbot_small/modeling_blenderbot_small.py b/src/transformers/models/blenderbot_small/modeling_blenderbot_small.py
index e0dcd95aa6..ad52947121 100755
--- a/src/transformers/models/blenderbot_small/modeling_blenderbot_small.py
+++ b/src/transformers/models/blenderbot_small/modeling_blenderbot_small.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" PyTorch BlenderbotSmall model. """
+""" PyTorch BlenderbotSmall model."""
import copy
@@ -382,7 +382,8 @@ class BlenderbotSmallDecoderLayer(nn.Module):
hidden_states (`torch.FloatTensor`): input to the layer of shape *(batch, seq_len, embed_dim)*
attention_mask (`torch.FloatTensor`): attention mask of size
*(batch, 1, tgt_len, src_len)* where padding elements are indicated by very large negative values.
- encoder_hidden_states (`torch.FloatTensor`): cross attention input to the layer of shape *(batch, seq_len, embed_dim)*
+ encoder_hidden_states (`torch.FloatTensor`):
+ cross attention input to the layer of shape *(batch, seq_len, embed_dim)*
encoder_attention_mask (`torch.FloatTensor`): encoder attention mask of size
*(batch, 1, tgt_len, src_len)* where padding elements are indicated by very large negative values.
layer_head_mask (`torch.FloatTensor`): mask for attention heads in a given layer of size
@@ -487,13 +488,13 @@ class BlenderbotSmallPreTrainedModel(PreTrainedModel):
BLENDERBOT_SMALL_START_DOCSTRING = r"""
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
Parameters:
config ([`BlenderbotSmallConfig`]):
@@ -505,28 +506,18 @@ BLENDERBOT_SMALL_START_DOCSTRING = r"""
BLENDERBOT_SMALL_GENERATION_EXAMPLE = r"""
Conversation example::
- >>> from transformers import BlenderbotSmallTokenizer, BlenderbotSmallForConditionalGeneration
- >>> mname = 'facebook/blenderbot_small-90M'
- >>> model = BlenderbotSmallForConditionalGeneration.from_pretrained(mname)
- >>> tokenizer = BlenderbotSmallTokenizer.from_pretrained(mname)
- >>> UTTERANCE = "My friends are cool but they eat too many carbs."
- >>> print("Human: ", UTTERANCE)
- >>> inputs = tokenizer([UTTERANCE], return_tensors='pt')
- >>> reply_ids = model.generate(**inputs)
- >>> print("Bot: ", tokenizer.batch_decode(reply_ids, skip_special_tokens=True)[0])
- what kind of carbs do they eat? i don't know much about carbs.
+ >>> from transformers import BlenderbotSmallTokenizer, BlenderbotSmallForConditionalGeneration >>> mname =
+ 'facebook/blenderbot_small-90M' >>> model = BlenderbotSmallForConditionalGeneration.from_pretrained(mname) >>>
+ tokenizer = BlenderbotSmallTokenizer.from_pretrained(mname) >>> UTTERANCE = "My friends are cool but they eat
+ too many carbs." >>> print("Human: ", UTTERANCE) >>> inputs = tokenizer([UTTERANCE], return_tensors='pt') >>>
+ reply_ids = model.generate(**inputs) >>> print("Bot: ", tokenizer.batch_decode(reply_ids,
+ skip_special_tokens=True)[0]) what kind of carbs do they eat? i don't know much about carbs.
- >>> REPLY = "I'm not sure"
- >>> print("Human: ", REPLY)
- >>> NEXT_UTTERANCE = (
- ... "My friends are cool but they eat too many carbs. "
- ... "what kind of carbs do they eat? i don't know much about carbs. "
- ... "I'm not sure."
- ... )
- >>> inputs = tokenizer([NEXT_UTTERANCE], return_tensors='pt')
- >>> inputs.pop("token_type_ids")
- >>> next_reply_ids = model.generate(**inputs)
- >>> print("Bot: ", tokenizer.batch_decode(next_reply_ids, skip_special_tokens=True)[0])
+ >>> REPLY = "I'm not sure" >>> print("Human: ", REPLY) >>> NEXT_UTTERANCE = ( ... "My friends are cool but they
+ eat too many carbs. " ... "what kind of carbs do they eat? i don't know much about carbs. " ...
+ "I'm not sure." ... ) >>> inputs = tokenizer([NEXT_UTTERANCE], return_tensors='pt') >>>
+ inputs.pop("token_type_ids") >>> next_reply_ids = model.generate(**inputs) >>> print("Bot: ",
+ tokenizer.batch_decode(next_reply_ids, skip_special_tokens=True)[0])
"""
BLENDERBOT_SMALL_INPUTS_DOCSTRING = r"""
@@ -535,9 +526,8 @@ BLENDERBOT_SMALL_INPUTS_DOCSTRING = r"""
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
it.
- Indices can be obtained using [`BlenderbotSmallTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`BlenderbotSmallTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -550,18 +540,17 @@ BLENDERBOT_SMALL_INPUTS_DOCSTRING = r"""
decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
Indices of decoder input sequence tokens in the vocabulary.
- Indices can be obtained using [`BlenderbotSmallTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`BlenderbotSmallTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are decoder input IDs?](../glossary#decoder-input-ids)
- BlenderbotSmall uses the `bos_token_id` as the starting token for `decoder_input_ids` generation.
- If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
+ BlenderbotSmall uses the `bos_token_id` as the starting token for `decoder_input_ids` generation. If
+ `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
`past_key_values`).
decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
- Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will
- also be used by default.
+ Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
+ be used by default.
head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*):
Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`:
@@ -575,39 +564,42 @@ BLENDERBOT_SMALL_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*):
- Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, 1]`:
+ Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0,
+ 1]`:
- 1 indicates the head is **not masked**,
- 0 indicates the head is **masked**.
encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*):
- Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*:
- `attentions`) `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`,
- *optional*) is a sequence of hidden-states at the output of the last layer of the encoder. Used in the
- cross-attention of the decoder.
+ Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`)
+ `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of
+ hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder.
past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors
- of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
- shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
+ Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
+ `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape
+ `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all ``decoder_input_ids``` of shape `(batch_size, sequence_length)`. inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ ``decoder_input_ids``` of shape `(batch_size, sequence_length)`. inputs_embeds (`torch.FloatTensor` of
+ shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids`
+ you can choose to directly pass an embedded representation. This is useful if you want more control over
+ how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup
+ matrix.
decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*):
Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded
- representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds`
- have to be input (see `past_key_values`). This is useful if you want more control over how to convert
+ representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be
+ input (see `past_key_values`). This is useful if you want more control over how to convert
`decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix.
- If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds`
- takes the value of `inputs_embeds`.
+ If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds` takes the value
+ of `inputs_embeds`.
use_cache (`bool`, *optional*):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`).
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`).
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -672,9 +664,8 @@ class BlenderbotSmallEncoder(BlenderbotSmallPreTrainedModel):
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
provide it.
- Indices can be obtained using [`BlenderbotSmallTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`]
- for details.
+ Indices can be obtained using [`BlenderbotSmallTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -691,9 +682,9 @@ class BlenderbotSmallEncoder(BlenderbotSmallPreTrainedModel):
- 0 indicates the head is **masked**.
inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded
- representation. This is useful if you want more control over how to convert `input_ids` indices
- into associated vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
+ This is useful if you want more control over how to convert `input_ids` indices into associated vectors
+ than the model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under
returned tensors for more detail.
@@ -789,8 +780,7 @@ class BlenderbotSmallEncoder(BlenderbotSmallPreTrainedModel):
class BlenderbotSmallDecoder(BlenderbotSmallPreTrainedModel):
"""
- Transformer decoder consisting of *config.decoder_layers* layers. Each layer is a
- [`BlenderbotSmallDecoderLayer`]
+ Transformer decoder consisting of *config.decoder_layers* layers. Each layer is a [`BlenderbotSmallDecoderLayer`]
Args:
config: BlenderbotSmallConfig
@@ -867,9 +857,8 @@ class BlenderbotSmallDecoder(BlenderbotSmallPreTrainedModel):
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
provide it.
- Indices can be obtained using [`BlenderbotSmallTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`]
- for details.
+ Indices can be obtained using [`BlenderbotSmallTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -904,19 +893,20 @@ class BlenderbotSmallDecoder(BlenderbotSmallPreTrainedModel):
- 0 indicates the head is **masked**.
past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2
- tensors of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional
- tensors of shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
+ Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of
+ shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
+ shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
Contains pre-computed hidden-states (key and values in the self-attention blocks and in the
- cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential
- decoding.
+ cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
- If `past_key_values` are used, the user can optionally input only the last
- `decoder_input_ids` (those that don't have their past key value states given to this model) of
- shape `(batch_size, 1)` instead of all ``decoder_input_ids``` of shape `(batch_size,
- sequence_length)`. inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices
- into associated vectors than the model's internal embedding lookup matrix.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those
+ that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of
+ all ``decoder_input_ids``` of shape `(batch_size, sequence_length)`. inputs_embeds (`torch.FloatTensor`
+ of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing
+ `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more
+ control over how to convert `input_ids` indices into associated vectors than the model's internal
+ embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under
returned tensors for more detail.
@@ -1257,7 +1247,8 @@ class BlenderbotSmallForConditionalGeneration(BlenderbotSmallPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
+ Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
+ config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
(masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
Returns:
@@ -1419,9 +1410,8 @@ class BlenderbotSmallForCausalLM(BlenderbotSmallPreTrainedModel):
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
provide it.
- Indices can be obtained using [`BlenderbotSmallTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`]
- for details.
+ Indices can be obtained using [`BlenderbotSmallTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -1450,25 +1440,24 @@ class BlenderbotSmallForCausalLM(BlenderbotSmallPreTrainedModel):
- 0 indicates the head is **masked**.
past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2
- tensors of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional
- tensors of shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. The two
- additional tensors are only required when the model is used as a decoder in a Sequence to Sequence
- model.
+ Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of
+ shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
+ shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. The two additional
+ tensors are only required when the model is used as a decoder in a Sequence to Sequence model.
Contains pre-computed hidden-states (key and values in the self-attention blocks and in the
- cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential
- decoding.
+ cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those
+ that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of
+ all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are
- ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
+ Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
+ config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
+ (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
use_cache (`bool`, *optional*):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`).
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
+ (see `past_key_values`).
- 1 for tokens that are **not masked**,
- 0 for tokens that are **masked**.
diff --git a/src/transformers/models/blenderbot_small/modeling_flax_blenderbot_small.py b/src/transformers/models/blenderbot_small/modeling_flax_blenderbot_small.py
index fd27a6c094..408589ab14 100644
--- a/src/transformers/models/blenderbot_small/modeling_flax_blenderbot_small.py
+++ b/src/transformers/models/blenderbot_small/modeling_flax_blenderbot_small.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" Flax BlenderbotSmall model. """
+""" Flax BlenderbotSmall model."""
import math
@@ -57,12 +57,13 @@ _CONFIG_FOR_DOC = "BlenderbotSmallConfig"
_TOKENIZER_FOR_DOC = "BlenderbotSmallTokenizer"
BLENDERBOT_SMALL_START_DOCSTRING = r"""
- This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading or saving, resizing the input
- embeddings, pruning heads etc.)
+ This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a Flax Linen [flax.nn.Module](https://flax.readthedocs.io/en/latest/_autosummary/flax.nn.module.html) subclass. Use it as a regular Flax
- Module and refer to the Flax documentation for all matter related to general usage and behavior.
+ This model is also a Flax Linen
+ [flax.nn.Module](https://flax.readthedocs.io/en/latest/_autosummary/flax.nn.module.html) subclass. Use it as a
+ regular Flax Module and refer to the Flax documentation for all matter related to general usage and behavior.
Finally, this model supports inherent JAX features such as:
@@ -74,11 +75,10 @@ BLENDERBOT_SMALL_START_DOCSTRING = r"""
Parameters:
config ([`BlenderbotSmallConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the
- model weights.
+ configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the model weights.
dtype (`jax.numpy.dtype`, *optional*, defaults to `jax.numpy.float32`):
- The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on
- GPUs) and `jax.numpy.bfloat16` (on TPUs).
+ The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on GPUs) and
+ `jax.numpy.bfloat16` (on TPUs).
This can be used to enable mixed-precision training or half-precision inference on GPUs or TPUs. If
specified all the computation will be performed with the given `dtype`.
@@ -86,8 +86,8 @@ BLENDERBOT_SMALL_START_DOCSTRING = r"""
**Note that this only specifies the dtype of the computation and does not influence the dtype of model
parameters.**
- If you wish to change the dtype of the model parameters, see
- [`~FlaxPreTrainedModel.to_fp16`] and [`~FlaxPreTrainedModel.to_bf16`].
+ If you wish to change the dtype of the model parameters, see [`~FlaxPreTrainedModel.to_fp16`] and
+ [`~FlaxPreTrainedModel.to_bf16`].
"""
BLENDERBOT_SMALL_INPUTS_DOCSTRING = r"""
@@ -96,9 +96,8 @@ BLENDERBOT_SMALL_INPUTS_DOCSTRING = r"""
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
it.
- Indices can be obtained using [`BlenderbotSmallTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`BlenderbotSmallTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`jnp.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
@@ -111,22 +110,23 @@ BLENDERBOT_SMALL_INPUTS_DOCSTRING = r"""
decoder_input_ids (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`, *optional*):
Indices of decoder input sequence tokens in the vocabulary.
- Indices can be obtained using [`BlenderbotSmallTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`BlenderbotSmallTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are decoder input IDs?](../glossary#decoder-input-ids)
For translation and summarization training, `decoder_input_ids` should be provided. If no
- `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to
- the right for denoising pre-training following the paper.
+ `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right
+ for denoising pre-training following the paper.
decoder_attention_mask (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`, *optional*):
- Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will
- also be used by default.
+ Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
+ be used by default.
- If you want to change padding behavior, you should modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy.
+ If you want to change padding behavior, you should modify to your needs. See diagram 1 in [the
+ paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy.
position_ids (`numpy.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
decoder_position_ids (`numpy.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the
range `[0, config.max_position_embeddings - 1]`.
@@ -147,9 +147,8 @@ BLENDERBOT_SMALL_ENCODE_INPUTS_DOCSTRING = r"""
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
it.
- Indices can be obtained using [`BlenderbotSmallTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`BlenderbotSmallTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`jnp.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
@@ -160,7 +159,8 @@ BLENDERBOT_SMALL_ENCODE_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
position_ids (`numpy.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -176,20 +176,18 @@ BLENDERBOT_SMALL_DECODE_INPUTS_DOCSTRING = r"""
decoder_input_ids (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`):
Indices of decoder input sequence tokens in the vocabulary.
- Indices can be obtained using [`BlenderbotSmallTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`BlenderbotSmallTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are decoder input IDs?](../glossary#decoder-input-ids)
For translation and summarization training, `decoder_input_ids` should be provided. If no
- `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to
- the right for denoising pre-training following the paper.
+ `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right
+ for denoising pre-training following the paper.
encoder_outputs (`tuple(tuple(jnp.ndarray)`):
- Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*:
- `attentions`) `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`,
- *optional*) is a sequence of hidden-states at the output of the last layer of the encoder. Used in the
- cross-attention of the decoder.
+ Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`)
+ `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of
+ hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder.
encoder_attention_mask (`jnp.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
@@ -198,10 +196,11 @@ BLENDERBOT_SMALL_DECODE_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
decoder_attention_mask (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`, *optional*):
- Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will
- also be used by default.
+ Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
+ be used by default.
- If you want to change padding behavior, you should modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy.
+ If you want to change padding behavior, you should modify to your needs. See diagram 1 in [the
+ paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy.
decoder_position_ids (`numpy.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the
range `[0, config.max_position_embeddings - 1]`.
@@ -941,9 +940,10 @@ class FlaxBlenderbotSmallPreTrainedModel(FlaxPreTrainedModel):
maximum possible length for auto-regressive decoding. Defines the sequence length of the initialized
cache.
encoder_outputs (`Union[FlaxBaseModelOutput, tuple(tuple(jnp.ndarray)]`):
- `encoder_outputs` consists of (`last_hidden_state`, *optional*: `hidden_states`,
- *optional*: `attentions`). `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of hidden-states at the output of the last layer of the
- encoder. Used in the cross-attention of the decoder.
+ `encoder_outputs` consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*:
+ `attentions`). `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*)
+ is a sequence of hidden-states at the output of the last layer of the encoder. Used in the
+ cross-attention of the decoder.
"""
# init input variables to retrieve cache
decoder_input_ids = jnp.ones((batch_size, max_length), dtype="i4")
@@ -1486,29 +1486,26 @@ FLAX_BLENDERBOT_SMALL_CONDITIONAL_GENERATION_DOCSTRING = """
>>> from transformers import BlenderbotSmallTokenizer, FlaxBlenderbotSmallForConditionalGeneration
- >>> model = FlaxBlenderbotSmallForConditionalGeneration.from_pretrained('facebook/blenderbot_small-90M')
- >>> tokenizer = BlenderbotSmallTokenizer.from_pretrained('facebook/blenderbot_small-90M')
+ >>> model = FlaxBlenderbotSmallForConditionalGeneration.from_pretrained('facebook/blenderbot_small-90M') >>>
+ tokenizer = BlenderbotSmallTokenizer.from_pretrained('facebook/blenderbot_small-90M')
- >>> ARTICLE_TO_SUMMARIZE = "My friends are cool but they eat too many carbs."
- >>> inputs = tokenizer([ARTICLE_TO_SUMMARIZE], max_length=1024, return_tensors='np')
+ >>> ARTICLE_TO_SUMMARIZE = "My friends are cool but they eat too many carbs." >>> inputs =
+ tokenizer([ARTICLE_TO_SUMMARIZE], max_length=1024, return_tensors='np')
- >>> # Generate Summary
- >>> summary_ids = model.generate(inputs['input_ids']).sequences
- >>> print(tokenizer.batch_decode(summary_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False))
+ >>> # Generate Summary >>> summary_ids = model.generate(inputs['input_ids']).sequences >>>
+ print(tokenizer.batch_decode(summary_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False))
Mask filling example::
- >>> from transformers import BlenderbotSmallTokenizer, FlaxBlenderbotSmallForConditionalGeneration
- >>> tokenizer = BlenderbotSmallTokenizer.from_pretrained('facebook/blenderbot_small-90M')
- >>> TXT = "My friends are but they eat too many carbs."
+ >>> from transformers import BlenderbotSmallTokenizer, FlaxBlenderbotSmallForConditionalGeneration >>>
+ tokenizer = BlenderbotSmallTokenizer.from_pretrained('facebook/blenderbot_small-90M') >>> TXT = "My friends are
+ but they eat too many carbs."
- >>> model = FlaxBlenderbotSmallForConditionalGeneration.from_pretrained('facebook/blenderbot_small-90M')
- >>> input_ids = tokenizer([TXT], return_tensors='np')['input_ids']
- >>> logits = model(input_ids).logits
+ >>> model = FlaxBlenderbotSmallForConditionalGeneration.from_pretrained('facebook/blenderbot_small-90M') >>>
+ input_ids = tokenizer([TXT], return_tensors='np')['input_ids'] >>> logits = model(input_ids).logits
- >>> masked_index = (input_ids[0] == tokenizer.mask_token_id).nonzero().item()
- >>> probs = jax.nn.softmax(logits[0, masked_index], axis=0)
- >>> values, predictions = jax.lax.top_k(probs)
+ >>> masked_index = (input_ids[0] == tokenizer.mask_token_id).nonzero().item() >>> probs =
+ jax.nn.softmax(logits[0, masked_index], axis=0) >>> values, predictions = jax.lax.top_k(probs)
>>> tokenizer.decode(predictions).split()
"""
diff --git a/src/transformers/models/blenderbot_small/modeling_tf_blenderbot_small.py b/src/transformers/models/blenderbot_small/modeling_tf_blenderbot_small.py
index dd12fa51d1..af8928f979 100644
--- a/src/transformers/models/blenderbot_small/modeling_tf_blenderbot_small.py
+++ b/src/transformers/models/blenderbot_small/modeling_tf_blenderbot_small.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" TF 2.0 BlenderbotSmall model. """
+""" TF 2.0 BlenderbotSmall model."""
import random
@@ -379,7 +379,8 @@ class TFBlenderbotSmallDecoderLayer(tf.keras.layers.Layer):
hidden_states (`tf.Tensor`): input to the layer of shape *(seq_len, batch, embed_dim)*
attention_mask (`tf.Tensor`): attention mask of size
*(batch, 1, tgt_len, src_len)* where padding elements are indicated by very large negative values.
- encoder_hidden_states (`tf.Tensor`): cross attention input to the layer of shape *(seq_len, batch, embed_dim)*
+ encoder_hidden_states (`tf.Tensor`):
+ cross attention input to the layer of shape *(seq_len, batch, embed_dim)*
encoder_attention_mask (`tf.Tensor`): encoder attention mask of size
*(batch, 1, tgt_len, src_len)* where padding elements are indicated by very large negative values.
layer_head_mask (`tf.Tensor`): mask for attention heads in a given layer of size
@@ -477,13 +478,13 @@ class TFBlenderbotSmallPreTrainedModel(TFPreTrainedModel):
BLENDERBOT_SMALL_START_DOCSTRING = r"""
- This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading or saving, resizing the input
- embeddings, pruning heads etc.)
+ This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use
- it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage
- and behavior.
+ This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it
+ as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and
+ behavior.
@@ -492,11 +493,11 @@ BLENDERBOT_SMALL_START_DOCSTRING = r"""
- having all inputs as keyword arguments (like PyTorch models), or
- having all inputs as a list, tuple or dict in the first positional arguments.
- This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all
- the tensors in the first argument of the model call function: `model(inputs)`.
+ This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all the
+ tensors in the first argument of the model call function: `model(inputs)`.
- If you choose this second option, there are three possibilities you can use to gather all the input Tensors in
- the first positional argument :
+ If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the
+ first positional argument :
- a single Tensor with `input_ids` only and nothing else: `model(input_ids)`
- a list of varying length with one or several input Tensors IN THE ORDER given in the docstring:
@@ -509,38 +510,29 @@ BLENDERBOT_SMALL_START_DOCSTRING = r"""
Args:
config ([`BlenderbotSmallConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~TFPreTrainedModel.from_pretrained`] method to load the
- model weights.
+ configuration. Check out the [`~TFPreTrainedModel.from_pretrained`] method to load the model weights.
"""
BLENDERBOT_SMALL_GENERATION_EXAMPLE = r"""
Conversation example::
- >>> from transformers import BlenderbotSmallTokenizer, TFBlenderbotSmallForConditionalGeneration
- >>> mname = 'facebook/blenderbot_small-90M'
- >>> model = BlenderbotSmallForConditionalGeneration.from_pretrained(mname)
- >>> tokenizer = BlenderbotSmallTokenizer.from_pretrained(mname)
+ >>> from transformers import BlenderbotSmallTokenizer, TFBlenderbotSmallForConditionalGeneration >>> mname =
+ 'facebook/blenderbot_small-90M' >>> model = BlenderbotSmallForConditionalGeneration.from_pretrained(mname) >>>
+ tokenizer = BlenderbotSmallTokenizer.from_pretrained(mname)
- >>> UTTERANCE = "My friends are cool but they eat too many carbs."
- >>> print("Human: ", UTTERANCE)
- >>> inputs = tokenizer([UTTERANCE], return_tensors='tf')
+ >>> UTTERANCE = "My friends are cool but they eat too many carbs." >>> print("Human: ", UTTERANCE) >>> inputs =
+ tokenizer([UTTERANCE], return_tensors='tf')
- >>> reply_ids = model.generate(**inputs)
- >>> print("Bot: ", tokenizer.batch_decode(reply_ids, skip_special_tokens=True)[0])
- what kind of carbs do they eat? i don't know much about carbs.
+ >>> reply_ids = model.generate(**inputs) >>> print("Bot: ", tokenizer.batch_decode(reply_ids,
+ skip_special_tokens=True)[0]) what kind of carbs do they eat? i don't know much about carbs.
- >>> REPLY = "I'm not sure"
- >>> print("Human: ", REPLY)
- >>> NEXT_UTTERANCE = (
- ... "My friends are cool but they eat too many carbs. "
- ... "what kind of carbs do they eat? i don't know much about carbs. "
- ... "I'm not sure."
- ... )
+ >>> REPLY = "I'm not sure" >>> print("Human: ", REPLY) >>> NEXT_UTTERANCE = ( ... "My friends are cool but they
+ eat too many carbs. " ... "what kind of carbs do they eat? i don't know much about carbs. " ...
+ "I'm not sure." ... )
- >>> inputs = tokenizer([NEXT_UTTERANCE], return_tensors='tf')
- >>> inputs.pop("token_type_ids")
- >>> next_reply_ids = model.generate(**inputs)
- >>> print("Bot: ", tokenizer.batch_decode(next_reply_ids, skip_special_tokens=True)[0])
+ >>> inputs = tokenizer([NEXT_UTTERANCE], return_tensors='tf') >>> inputs.pop("token_type_ids") >>>
+ next_reply_ids = model.generate(**inputs) >>> print("Bot: ", tokenizer.batch_decode(next_reply_ids,
+ skip_special_tokens=True)[0])
"""
BLENDERBOT_SMALL_INPUTS_DOCSTRING = r"""
@@ -548,9 +540,8 @@ BLENDERBOT_SMALL_INPUTS_DOCSTRING = r"""
input_ids (`tf.Tensor` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`BlenderbotSmallTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`BlenderbotSmallTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`tf.Tensor` of shape `({0})`, *optional*):
@@ -563,14 +554,13 @@ BLENDERBOT_SMALL_INPUTS_DOCSTRING = r"""
decoder_input_ids (`tf.Tensor` of shape `(batch_size, target_sequence_length)`, *optional*):
Indices of decoder input sequence tokens in the vocabulary.
- Indices can be obtained using [`BlenderbotSmallTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`BlenderbotSmallTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are decoder input IDs?](../glossary#decoder-input-ids)
- BlenderbotSmall uses the `bos_token_id` as the starting token for `decoder_input_ids` generation.
- If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
+ BlenderbotSmall uses the `bos_token_id` as the starting token for `decoder_input_ids` generation. If
+ `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
`past_key_values`).
decoder_attention_mask (`tf.Tensor` of shape `(batch_size, target_sequence_length)`, *optional*):
will be made by default and ignore pad tokens. It is not recommended to set this for most use cases.
@@ -597,12 +587,12 @@ BLENDERBOT_SMALL_INPUTS_DOCSTRING = r"""
of shape `(batch_size, sequence_length, hidden_size)` is a sequence of
past_key_values (`Tuple[Tuple[tf.Tensor]]` of length `config.n_layers`)
contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ `decoder_input_ids` of shape `(batch_size, sequence_length)`.
use_cache (`bool`, *optional*, defaults to `True`):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`). Set to `False` during training, `True` during generation
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`). Set to `False` during training, `True` during generation
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the
@@ -612,8 +602,8 @@ BLENDERBOT_SMALL_INPUTS_DOCSTRING = r"""
more detail. This argument can be used only in eager mode, in graph mode the value in the config will be
used instead.
return_dict (`bool`, *optional*):
- Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This
- argument can be used in eager mode, in graph mode the value will always be set to True.
+ Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This argument can be used
+ in eager mode, in graph mode the value will always be set to True.
training (`bool`, *optional*, defaults to `False`):
Whether or not to use the model in training mode (some modules like dropout modules have different
behaviors between training and evaluation).
@@ -673,9 +663,8 @@ class TFBlenderbotSmallEncoder(tf.keras.layers.Layer):
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
provide it.
- Indices can be obtained using [`BlenderbotSmallTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`]
- for details.
+ Indices can be obtained using [`BlenderbotSmallTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -685,15 +674,16 @@ class TFBlenderbotSmallEncoder(tf.keras.layers.Layer):
- 0 for tokens that are **masked**.
[What are attention masks?](../glossary#attention-mask)
- head_mask (`tf.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, `optional): Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`:
+ head_mask (`tf.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, `optional):
+ Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`:
- 1 indicates the head is **not masked**,
- 0 indicates the head is **masked**.
inputs_embeds (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded
- representation. This is useful if you want more control over how to convert `input_ids` indices
- into associated vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
+ This is useful if you want more control over how to convert `input_ids` indices into associated vectors
+ than the model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under
returned tensors for more detail. This argument can be used only in eager mode, in graph mode the value
@@ -703,8 +693,8 @@ class TFBlenderbotSmallEncoder(tf.keras.layers.Layer):
for more detail. This argument can be used only in eager mode, in graph mode the value in the config
will be used instead.
return_dict (`bool`, *optional*):
- Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This
- argument can be used in eager mode, in graph mode the value will always be set to True.
+ Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This argument can be
+ used in eager mode, in graph mode the value will always be set to True.
training (`bool`, *optional*, defaults to `False`):
Whether or not to use the model in training mode (some modules like dropout modules have different
behaviors between training and evaluation).
@@ -793,8 +783,7 @@ class TFBlenderbotSmallEncoder(tf.keras.layers.Layer):
class TFBlenderbotSmallDecoder(tf.keras.layers.Layer):
config_class = BlenderbotSmallConfig
"""
- Transformer decoder consisting of *config.decoder_layers* layers. Each layer is a
- [`TFBlenderbotSmallDecoderLayer`]
+ Transformer decoder consisting of *config.decoder_layers* layers. Each layer is a [`TFBlenderbotSmallDecoderLayer`]
Args:
config: BlenderbotSmallConfig
@@ -847,9 +836,8 @@ class TFBlenderbotSmallDecoder(tf.keras.layers.Layer):
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
provide it.
- Indices can be obtained using [`BlenderbotSmallTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`]
- for details.
+ Indices can be obtained using [`BlenderbotSmallTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -886,11 +874,13 @@ class TFBlenderbotSmallDecoder(tf.keras.layers.Layer):
Contains precomputed key and value hidden-states of the attention blocks. Can be used to speed up
decoding.
- If `past_key_values` are used, the user can optionally input only the last
- `decoder_input_ids` (those that don't have their past key value states given to this model) of
- shape `(batch_size, 1)` instead of all ``decoder_input_ids``` of shape `(batch_size,
- sequence_length)`. inputs_embeds (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices
- into associated vectors than the model's internal embedding lookup matrix.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those
+ that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of
+ all ``decoder_input_ids``` of shape `(batch_size, sequence_length)`. inputs_embeds (`tf.Tensor` of
+ shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing
+ `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more
+ control over how to convert `input_ids` indices into associated vectors than the model's internal
+ embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under
returned tensors for more detail. This argument can be used only in eager mode, in graph mode the value
@@ -900,8 +890,8 @@ class TFBlenderbotSmallDecoder(tf.keras.layers.Layer):
for more detail. This argument can be used only in eager mode, in graph mode the value in the config
will be used instead.
return_dict (`bool`, *optional*):
- Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This
- argument can be used in eager mode, in graph mode the value will always be set to True.
+ Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This argument can be
+ used in eager mode, in graph mode the value will always be set to True.
training (`bool`, *optional*, defaults to `False`):
Whether or not to use the model in training mode (some modules like dropout modules have different
behaviors between training and evaluation).
@@ -1343,7 +1333,8 @@ class TFBlenderbotSmallForConditionalGeneration(TFBlenderbotSmallPreTrainedModel
):
r"""
labels (`tf.tensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
+ Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
+ config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
(masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
Returns:
diff --git a/src/transformers/models/blenderbot_small/tokenization_blenderbot_small.py b/src/transformers/models/blenderbot_small/tokenization_blenderbot_small.py
index 29746559be..f5263a5af9 100644
--- a/src/transformers/models/blenderbot_small/tokenization_blenderbot_small.py
+++ b/src/transformers/models/blenderbot_small/tokenization_blenderbot_small.py
@@ -68,8 +68,8 @@ class BlenderbotSmallTokenizer(PreTrainedTokenizer):
"""
Constructs a Blenderbot-90M tokenizer based on BPE (Byte-Pair-Encoding)
- This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods.
- Users should refer to the superclass for more information regarding methods.
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
+ the superclass for more information regarding methods.
Args:
vocab_file (`str`):
@@ -202,7 +202,7 @@ class BlenderbotSmallTokenizer(PreTrainedTokenizer):
return self.decoder.get(index, self.unk_token)
def convert_tokens_to_string(self, tokens: List[str]) -> str:
- """Converts a sequence of tokens in a single string."""
+ """Converts a sequence of tokens in a single string."""
out_string = " ".join(tokens).replace("@@ ", "").strip()
return out_string
diff --git a/src/transformers/models/byt5/tokenization_byt5.py b/src/transformers/models/byt5/tokenization_byt5.py
index 7c0e94f35c..4846b58aa2 100644
--- a/src/transformers/models/byt5/tokenization_byt5.py
+++ b/src/transformers/models/byt5/tokenization_byt5.py
@@ -29,8 +29,8 @@ class ByT5Tokenizer(PreTrainedTokenizer):
"""
Construct a ByT5 tokenizer. ByT5 simply uses raw bytes utf-8 encoding.
- This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods.
- Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
+ this superclass for more information regarding those methods.
Args:
eos_token (`str`, *optional*, defaults to `""`):
@@ -38,8 +38,8 @@ class ByT5Tokenizer(PreTrainedTokenizer):
- When building a sequence using special tokens, this is not the token that is used for the end of
- sequence. The token used is the `sep_token`.
+ When building a sequence using special tokens, this is not the token that is used for the end of sequence.
+ The token used is the `sep_token`.
@@ -52,7 +52,8 @@ class ByT5Tokenizer(PreTrainedTokenizer):
Add a number of extra ids added to the end of the vocabulary for use as sentinels. These tokens are
accessible as "" where "{%d}" is a number between 0 and extra_ids-1. Extra tokens are
indexed from the end of the vocabulary up to beginning ("" is the last token in the vocabulary
- like in ByT5 preprocessing see [here](https://github.com/google-research/text-to-text-transfer-transformer/blob/9fd7b14a769417be33bc6c850f9598764913c833/t5/data/preprocessors.py#L2117)).
+ like in ByT5 preprocessing see
+ [here](https://github.com/google-research/text-to-text-transfer-transformer/blob/9fd7b14a769417be33bc6c850f9598764913c833/t5/data/preprocessors.py#L2117)).
additional_special_tokens (`List[str]`, *optional*):
Additional special tokens used by the tokenizer.
"""
diff --git a/src/transformers/models/camembert/configuration_camembert.py b/src/transformers/models/camembert/configuration_camembert.py
index 2479ca98a6..a65ebd7c44 100644
--- a/src/transformers/models/camembert/configuration_camembert.py
+++ b/src/transformers/models/camembert/configuration_camembert.py
@@ -13,7 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" CamemBERT configuration """
+""" CamemBERT configuration"""
from collections import OrderedDict
from typing import Mapping
@@ -34,8 +34,8 @@ CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class CamembertConfig(RobertaConfig):
"""
- This class overrides [`RobertaConfig`]. Please check the superclass for the appropriate
- documentation alongside usage examples.
+ This class overrides [`RobertaConfig`]. Please check the superclass for the appropriate documentation alongside
+ usage examples.
"""
model_type = "camembert"
diff --git a/src/transformers/models/camembert/modeling_camembert.py b/src/transformers/models/camembert/modeling_camembert.py
index 872c936861..b1a10319d6 100644
--- a/src/transformers/models/camembert/modeling_camembert.py
+++ b/src/transformers/models/camembert/modeling_camembert.py
@@ -13,7 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-"""PyTorch CamemBERT model. """
+"""PyTorch CamemBERT model."""
from ...file_utils import add_start_docstrings
from ...utils import logging
@@ -42,19 +42,18 @@ CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_LIST = [
CAMEMBERT_START_DOCSTRING = r"""
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
Parameters:
config ([`CamembertConfig`]): Model configuration class with all the parameters of the
model. Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
@@ -64,21 +63,21 @@ CAMEMBERT_START_DOCSTRING = r"""
)
class CamembertModel(RobertaModel):
"""
- This class overrides [`RobertaModel`]. Please check the superclass for the appropriate
- documentation alongside usage examples.
+ This class overrides [`RobertaModel`]. Please check the superclass for the appropriate documentation alongside
+ usage examples.
"""
config_class = CamembertConfig
@add_start_docstrings(
- """CamemBERT Model with a `language modeling` head on top. """,
+ """CamemBERT Model with a `language modeling` head on top.""",
CAMEMBERT_START_DOCSTRING,
)
class CamembertForMaskedLM(RobertaForMaskedLM):
"""
- This class overrides [`RobertaForMaskedLM`]. Please check the superclass for the appropriate
- documentation alongside usage examples.
+ This class overrides [`RobertaForMaskedLM`]. Please check the superclass for the appropriate documentation
+ alongside usage examples.
"""
config_class = CamembertConfig
@@ -93,8 +92,8 @@ class CamembertForMaskedLM(RobertaForMaskedLM):
)
class CamembertForSequenceClassification(RobertaForSequenceClassification):
"""
- This class overrides [`RobertaForSequenceClassification`]. Please check the superclass for the
- appropriate documentation alongside usage examples.
+ This class overrides [`RobertaForSequenceClassification`]. Please check the superclass for the appropriate
+ documentation alongside usage examples.
"""
config_class = CamembertConfig
@@ -109,8 +108,8 @@ class CamembertForSequenceClassification(RobertaForSequenceClassification):
)
class CamembertForMultipleChoice(RobertaForMultipleChoice):
"""
- This class overrides [`RobertaForMultipleChoice`]. Please check the superclass for the
- appropriate documentation alongside usage examples.
+ This class overrides [`RobertaForMultipleChoice`]. Please check the superclass for the appropriate documentation
+ alongside usage examples.
"""
config_class = CamembertConfig
@@ -125,8 +124,8 @@ class CamembertForMultipleChoice(RobertaForMultipleChoice):
)
class CamembertForTokenClassification(RobertaForTokenClassification):
"""
- This class overrides [`RobertaForTokenClassification`]. Please check the superclass for the
- appropriate documentation alongside usage examples.
+ This class overrides [`RobertaForTokenClassification`]. Please check the superclass for the appropriate
+ documentation alongside usage examples.
"""
config_class = CamembertConfig
@@ -141,20 +140,20 @@ class CamembertForTokenClassification(RobertaForTokenClassification):
)
class CamembertForQuestionAnswering(RobertaForQuestionAnswering):
"""
- This class overrides [`RobertaForQuestionAnswering`]. Please check the superclass for the
- appropriate documentation alongside usage examples.
+ This class overrides [`RobertaForQuestionAnswering`]. Please check the superclass for the appropriate documentation
+ alongside usage examples.
"""
config_class = CamembertConfig
@add_start_docstrings(
- """CamemBERT Model with a `language modeling` head on top for CLM fine-tuning. """, CAMEMBERT_START_DOCSTRING
+ """CamemBERT Model with a `language modeling` head on top for CLM fine-tuning.""", CAMEMBERT_START_DOCSTRING
)
class CamembertForCausalLM(RobertaForCausalLM):
"""
- This class overrides [`RobertaForCausalLM`]. Please check the superclass for the appropriate
- documentation alongside usage examples.
+ This class overrides [`RobertaForCausalLM`]. Please check the superclass for the appropriate documentation
+ alongside usage examples.
"""
config_class = CamembertConfig
diff --git a/src/transformers/models/camembert/modeling_tf_camembert.py b/src/transformers/models/camembert/modeling_tf_camembert.py
index bf17be951e..b46246465b 100644
--- a/src/transformers/models/camembert/modeling_tf_camembert.py
+++ b/src/transformers/models/camembert/modeling_tf_camembert.py
@@ -13,7 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" TF 2.0 CamemBERT model. """
+""" TF 2.0 CamemBERT model."""
from ...file_utils import add_start_docstrings
from ...utils import logging
@@ -37,13 +37,13 @@ TF_CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_LIST = [
CAMEMBERT_START_DOCSTRING = r"""
- This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading or saving, resizing the input
- embeddings, pruning heads etc.)
+ This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use
- it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage
- and behavior.
+ This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it
+ as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and
+ behavior.
@@ -52,11 +52,11 @@ CAMEMBERT_START_DOCSTRING = r"""
- having all inputs as keyword arguments (like PyTorch models), or
- having all inputs as a list, tuple or dict in the first positional arguments.
- This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all
- the tensors in the first argument of the model call function: `model(inputs)`.
+ This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all the
+ tensors in the first argument of the model call function: `model(inputs)`.
- If you choose this second option, there are three possibilities you can use to gather all the input Tensors in
- the first positional argument :
+ If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the
+ first positional argument :
- a single Tensor with `input_ids` only and nothing else: `model(inputs_ids)`
- a list of varying length with one or several input Tensors IN THE ORDER given in the docstring:
@@ -69,8 +69,7 @@ CAMEMBERT_START_DOCSTRING = r"""
Parameters:
config ([`CamembertConfig`]): Model configuration class with all the parameters of the
model. Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
@@ -80,21 +79,21 @@ CAMEMBERT_START_DOCSTRING = r"""
)
class TFCamembertModel(TFRobertaModel):
"""
- This class overrides [`TFRobertaModel`]. Please check the superclass for the appropriate
- documentation alongside usage examples.
+ This class overrides [`TFRobertaModel`]. Please check the superclass for the appropriate documentation alongside
+ usage examples.
"""
config_class = CamembertConfig
@add_start_docstrings(
- """CamemBERT Model with a `language modeling` head on top. """,
+ """CamemBERT Model with a `language modeling` head on top.""",
CAMEMBERT_START_DOCSTRING,
)
class TFCamembertForMaskedLM(TFRobertaForMaskedLM):
"""
- This class overrides [`TFRobertaForMaskedLM`]. Please check the superclass for the appropriate
- documentation alongside usage examples.
+ This class overrides [`TFRobertaForMaskedLM`]. Please check the superclass for the appropriate documentation
+ alongside usage examples.
"""
config_class = CamembertConfig
@@ -109,8 +108,8 @@ class TFCamembertForMaskedLM(TFRobertaForMaskedLM):
)
class TFCamembertForSequenceClassification(TFRobertaForSequenceClassification):
"""
- This class overrides [`TFRobertaForSequenceClassification`]. Please check the superclass for the
- appropriate documentation alongside usage examples.
+ This class overrides [`TFRobertaForSequenceClassification`]. Please check the superclass for the appropriate
+ documentation alongside usage examples.
"""
config_class = CamembertConfig
@@ -125,8 +124,8 @@ class TFCamembertForSequenceClassification(TFRobertaForSequenceClassification):
)
class TFCamembertForTokenClassification(TFRobertaForTokenClassification):
"""
- This class overrides [`TFRobertaForTokenClassification`]. Please check the superclass for the
- appropriate documentation alongside usage examples.
+ This class overrides [`TFRobertaForTokenClassification`]. Please check the superclass for the appropriate
+ documentation alongside usage examples.
"""
config_class = CamembertConfig
@@ -141,8 +140,8 @@ class TFCamembertForTokenClassification(TFRobertaForTokenClassification):
)
class TFCamembertForMultipleChoice(TFRobertaForMultipleChoice):
"""
- This class overrides [`TFRobertaForMultipleChoice`]. Please check the superclass for the
- appropriate documentation alongside usage examples.
+ This class overrides [`TFRobertaForMultipleChoice`]. Please check the superclass for the appropriate documentation
+ alongside usage examples.
"""
config_class = CamembertConfig
@@ -157,8 +156,8 @@ class TFCamembertForMultipleChoice(TFRobertaForMultipleChoice):
)
class TFCamembertForQuestionAnswering(TFRobertaForQuestionAnswering):
"""
- This class overrides [`TFRobertaForQuestionAnswering`]. Please check the superclass for the
- appropriate documentation alongside usage examples.
+ This class overrides [`TFRobertaForQuestionAnswering`]. Please check the superclass for the appropriate
+ documentation alongside usage examples.
"""
config_class = CamembertConfig
diff --git a/src/transformers/models/camembert/tokenization_camembert.py b/src/transformers/models/camembert/tokenization_camembert.py
index 2bb76236ee..2d21c2e5bb 100644
--- a/src/transformers/models/camembert/tokenization_camembert.py
+++ b/src/transformers/models/camembert/tokenization_camembert.py
@@ -44,11 +44,11 @@ SPIECE_UNDERLINE = "▁"
class CamembertTokenizer(PreTrainedTokenizer):
"""
- Adapted from [`RobertaTokenizer`] and [`XLNetTokenizer`]. Construct a
- CamemBERT tokenizer. Based on [SentencePiece](https://github.com/google/sentencepiece).
+ Adapted from [`RobertaTokenizer`] and [`XLNetTokenizer`]. Construct a CamemBERT tokenizer. Based on
+ [SentencePiece](https://github.com/google/sentencepiece).
- This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods.
- Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
+ this superclass for more information regarding those methods.
Args:
vocab_file (`str`):
@@ -69,8 +69,8 @@ class CamembertTokenizer(PreTrainedTokenizer):
- When building a sequence using special tokens, this is not the token that is used for the end of
- sequence. The token used is the `sep_token`.
+ When building a sequence using special tokens, this is not the token that is used for the end of sequence.
+ The token used is the `sep_token`.
@@ -92,7 +92,9 @@ class CamembertTokenizer(PreTrainedTokenizer):
additional_special_tokens (`List[str]`, *optional*, defaults to `["NOTUSED", "NOTUSED"]`):
Additional special tokens used by the tokenizer.
sp_model_kwargs (`dict`, *optional*):
- Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things, to set:
+ Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for
+ SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things,
+ to set:
- `enable_sampling`: Enable subword regularization.
- `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout.
diff --git a/src/transformers/models/camembert/tokenization_camembert_fast.py b/src/transformers/models/camembert/tokenization_camembert_fast.py
index 782ba2f5c3..d9dfa5f417 100644
--- a/src/transformers/models/camembert/tokenization_camembert_fast.py
+++ b/src/transformers/models/camembert/tokenization_camembert_fast.py
@@ -54,10 +54,11 @@ SPIECE_UNDERLINE = "▁"
class CamembertTokenizerFast(PreTrainedTokenizerFast):
"""
Construct a "fast" CamemBERT tokenizer (backed by HuggingFace's *tokenizers* library). Adapted from
- [`RobertaTokenizer`] and [`XLNetTokenizer`]. Based on [BPE](https://huggingface.co/docs/tokenizers/python/latest/components.html?highlight=BPE#models).
+ [`RobertaTokenizer`] and [`XLNetTokenizer`]. Based on
+ [BPE](https://huggingface.co/docs/tokenizers/python/latest/components.html?highlight=BPE#models).
- This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main
- methods. Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should
+ refer to this superclass for more information regarding those methods.
Args:
vocab_file (`str`):
@@ -78,8 +79,8 @@ class CamembertTokenizerFast(PreTrainedTokenizerFast):
- When building a sequence using special tokens, this is not the token that is used for the end of
- sequence. The token used is the `sep_token`.
+ When building a sequence using special tokens, this is not the token that is used for the end of sequence.
+ The token used is the `sep_token`.
diff --git a/src/transformers/models/canine/configuration_canine.py b/src/transformers/models/canine/configuration_canine.py
index b57a4fafff..383b371445 100644
--- a/src/transformers/models/canine/configuration_canine.py
+++ b/src/transformers/models/canine/configuration_canine.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" CANINE model configuration """
+""" CANINE model configuration"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging
@@ -28,12 +28,13 @@ CANINE_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class CanineConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`CanineModel`]. It is used to
- instantiate an CANINE model according to the specified arguments, defining the model architecture. Instantiating a
- configuration with the defaults will yield a similar configuration to that of the CANINE [google/canine-s](https://huggingface.co/google/canine-s) architecture.
+ This is the configuration class to store the configuration of a [`CanineModel`]. It is used to instantiate an
+ CANINE model according to the specified arguments, defining the model architecture. Instantiating a configuration
+ with the defaults will yield a similar configuration to that of the CANINE
+ [google/canine-s](https://huggingface.co/google/canine-s) architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
@@ -46,8 +47,8 @@ class CanineConfig(PretrainedConfig):
intermediate_size (`int`, *optional*, defaults to 3072):
Dimension of the "intermediate" (i.e., feed-forward) layer in the Transformer encoders.
hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"selu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"selu"` and `"gelu_new"` are supported.
hidden_dropout_prob (`float`, *optional*, defaults to 0.1):
The dropout probabilitiy for all fully connected layers in the embeddings, encoders, and pooler.
attention_probs_dropout_prob (`float`, *optional*, defaults to 0.1):
diff --git a/src/transformers/models/canine/modeling_canine.py b/src/transformers/models/canine/modeling_canine.py
index 7d87a7b855..86aff3b590 100644
--- a/src/transformers/models/canine/modeling_canine.py
+++ b/src/transformers/models/canine/modeling_canine.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" PyTorch CANINE model. """
+""" PyTorch CANINE model."""
import copy
@@ -65,9 +65,8 @@ _PRIMES = [31, 43, 59, 61, 73, 97, 103, 113, 137, 149, 157, 173, 181, 193, 211,
@dataclass
class CanineModelOutputWithPooling(ModelOutput):
"""
- Output type of [`CanineModel`]. Based on
- [`~modeling_outputs.BaseModelOutputWithPooling`], but with slightly different
- `hidden_states` and `attentions`, as these also include the hidden states and attentions of the shallow
+ Output type of [`CanineModel`]. Based on [`~modeling_outputs.BaseModelOutputWithPooling`], but with slightly
+ different `hidden_states` and `attentions`, as these also include the hidden states and attentions of the shallow
Transformer encoders.
Args:
@@ -79,15 +78,17 @@ class CanineModelOutputWithPooling(ModelOutput):
Transformer encoder, further processed by a Linear layer and a Tanh activation function. The Linear layer
weights are trained from the next sentence prediction (classification) objective during pretraining.
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the input to each encoder + one for the output of each layer of
- each encoder) of shape `(batch_size, sequence_length, hidden_size)` and `(batch_size, sequence_length // config.downsampling_rate, hidden_size)`. Hidden-states of the model at the output of
- each layer plus the initial input to each Transformer encoder. The hidden states of the shallow encoders
- have length `sequence_length`, but the hidden states of the deep encoder have length
- `sequence_length` // `config.downsampling_rate`.
+ Tuple of `torch.FloatTensor` (one for the input to each encoder + one for the output of each layer of each
+ encoder) of shape `(batch_size, sequence_length, hidden_size)` and `(batch_size, sequence_length //
+ config.downsampling_rate, hidden_size)`. Hidden-states of the model at the output of each layer plus the
+ initial input to each Transformer encoder. The hidden states of the shallow encoders have length
+ `sequence_length`, but the hidden states of the deep encoder have length `sequence_length` //
+ `config.downsampling_rate`.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of the 3 Transformer encoders of shape
- `(batch_size, num_heads, sequence_length, sequence_length)` and `(batch_size, num_heads, sequence_length // config.downsampling_rate, sequence_length // config.downsampling_rate)`. Attentions
- weights after the attention softmax, used to compute the weighted average in the self-attention heads.
+ Tuple of `torch.FloatTensor` (one for each layer) of the 3 Transformer encoders of shape `(batch_size,
+ num_heads, sequence_length, sequence_length)` and `(batch_size, num_heads, sequence_length //
+ config.downsampling_rate, sequence_length // config.downsampling_rate)`. Attentions weights after the
+ attention softmax, used to compute the weighted average in the self-attention heads.
"""
last_hidden_state: torch.FloatTensor = None
@@ -515,16 +516,16 @@ class CanineAttention(nn.Module):
Additional arguments related to local attention:
- **local** (`bool`, *optional*, defaults to `False`) -- Whether to apply local attention.
- - **always_attend_to_first_position** (`bool`, *optional*, defaults to `False`) -- Should all blocks
- be able to attend
- to the `to_tensor`'s first position (e.g. a [CLS] position)? - **first_position_attends_to_all**
- (`bool`, *optional*, defaults to `False`) -- Should the *from_tensor*'s first position be able to
- attend to all positions within the *from_tensor*? - **attend_from_chunk_width** (`int`, *optional*,
- defaults to 128) -- The width of each block-wise chunk in `from_tensor`. - **attend_from_chunk_stride**
- (`int`, *optional*, defaults to 128) -- The number of elements to skip when moving to the next block in
- `from_tensor`. - **attend_to_chunk_width** (`int`, *optional*, defaults to 128) -- The width of each
- block-wise chunk in *to_tensor*. - **attend_to_chunk_stride** (`int`, *optional*, defaults to 128) -- The
- number of elements to skip when moving to the next block in `to_tensor`.
+ - **always_attend_to_first_position** (`bool`, *optional*, defaults to `False`) -- Should all blocks be able to
+ attend
+ to the `to_tensor`'s first position (e.g. a [CLS] position)? - **first_position_attends_to_all** (`bool`,
+ *optional*, defaults to `False`) -- Should the *from_tensor*'s first position be able to attend to all
+ positions within the *from_tensor*? - **attend_from_chunk_width** (`int`, *optional*, defaults to 128) -- The
+ width of each block-wise chunk in `from_tensor`. - **attend_from_chunk_stride** (`int`, *optional*, defaults to
+ 128) -- The number of elements to skip when moving to the next block in `from_tensor`. -
+ **attend_to_chunk_width** (`int`, *optional*, defaults to 128) -- The width of each block-wise chunk in
+ *to_tensor*. - **attend_to_chunk_stride** (`int`, *optional*, defaults to 128) -- The number of elements to
+ skip when moving to the next block in `to_tensor`.
"""
def __init__(
@@ -926,8 +927,7 @@ CANINE_START_DOCSTRING = r"""
Parameters:
config ([`CanineConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
CANINE_INPUTS_DOCSTRING = r"""
@@ -935,9 +935,8 @@ CANINE_INPUTS_DOCSTRING = r"""
input_ids (`torch.LongTensor` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`CanineTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`CanineTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
@@ -948,14 +947,16 @@ CANINE_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
@@ -965,9 +966,9 @@ CANINE_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert *input_ids* indices into associated vectors
- than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -1295,8 +1296,9 @@ class CanineForSequenceClassification(CaninePreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1391,7 +1393,8 @@ class CanineForMultipleChoice(CaninePreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
+ Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
+ num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
`input_ids` above)
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1572,12 +1575,12 @@ class CanineForQuestionAnswering(CaninePreTrainedModel):
r"""
start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the start of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the end of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
diff --git a/src/transformers/models/canine/tokenization_canine.py b/src/transformers/models/canine/tokenization_canine.py
index 4bf96f1828..fba01e03c0 100644
--- a/src/transformers/models/canine/tokenization_canine.py
+++ b/src/transformers/models/canine/tokenization_canine.py
@@ -67,8 +67,7 @@ class CanineTokenizer(PreTrainedTokenizer):
[`CanineTokenizer`] inherits from [`PreTrainedTokenizer`].
- Refer to superclass [`PreTrainedTokenizer`] for usage examples and documentation concerning
- parameters.
+ Refer to superclass [`PreTrainedTokenizer`] for usage examples and documentation concerning parameters.
Args:
model_max_length (`int`, *optional*, defaults to 2048):
@@ -229,8 +228,7 @@ class CanineTokenizer(PreTrainedTokenizer):
Optional second list of IDs for sequence pairs.
Returns:
- `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given
- sequence(s).
+ `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
"""
sep = [self.sep_token_id]
cls = [self.cls_token_id]
diff --git a/src/transformers/models/clip/configuration_clip.py b/src/transformers/models/clip/configuration_clip.py
index e0a34e722d..3be63adfb4 100644
--- a/src/transformers/models/clip/configuration_clip.py
+++ b/src/transformers/models/clip/configuration_clip.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" CLIP model configuration """
+""" CLIP model configuration"""
import copy
@@ -30,13 +30,13 @@ CLIP_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class CLIPTextConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`CLIPModel`]. It is used to
- instantiate an CLIP model according to the specified arguments, defining the model architecture. Instantiating a
- configuration with the defaults will yield a similar configuration to that of the CLIP
+ This is the configuration class to store the configuration of a [`CLIPModel`]. It is used to instantiate an CLIP
+ model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
+ defaults will yield a similar configuration to that of the CLIP
[openai/clip-vit-base-patch32](https://huggingface.co/openai/clip-vit-base-patch32) architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
@@ -55,9 +55,9 @@ class CLIPTextConfig(PretrainedConfig):
The maximum sequence length that this model might ever be used with. Typically set this to something large
just in case (e.g., 512 or 1024 or 2048).
hidden_act (`str` or `function`, *optional*, defaults to `"quick_gelu"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"selu"` and `"gelu_new"` ``"quick_gelu"` are supported. layer_norm_eps (`float`, *optional*, defaults to 1e-5):
- The epsilon used by the layer normalization layers.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"selu"` and `"gelu_new"` ``"quick_gelu"` are supported. layer_norm_eps (`float`, *optional*,
+ defaults to 1e-5): The epsilon used by the layer normalization layers.
attention_dropout (`float`, *optional*, defaults to 0.0):
The dropout ratio for the attention probabilities.
dropout (`float`, *optional*, defaults to 0.0):
@@ -121,13 +121,13 @@ class CLIPTextConfig(PretrainedConfig):
class CLIPVisionConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`CLIPModel`]. It is used to
- instantiate an CLIP model according to the specified arguments, defining the model architecture. Instantiating a
- configuration with the defaults will yield a similar configuration to that of the CLIP
+ This is the configuration class to store the configuration of a [`CLIPModel`]. It is used to instantiate an CLIP
+ model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
+ defaults will yield a similar configuration to that of the CLIP
[openai/clip-vit-base-patch32](https://huggingface.co/openai/clip-vit-base-patch32) architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
@@ -144,9 +144,9 @@ class CLIPVisionConfig(PretrainedConfig):
patch_size (`int`, *optional*, defaults to 32):
The size (resolution) of each patch.
hidden_act (`str` or `function`, *optional*, defaults to `"quick_gelu"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"selu"` and `"gelu_new"` ``"quick_gelu"` are supported. layer_norm_eps (`float`, *optional*, defaults to 1e-5):
- The epsilon used by the layer normalization layers.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"selu"` and `"gelu_new"` ``"quick_gelu"` are supported. layer_norm_eps (`float`, *optional*,
+ defaults to 1e-5): The epsilon used by the layer normalization layers.
dropout (`float`, *optional*, defaults to 0.0):
The dropout probabilitiy for all fully connected layers in the embeddings, encoder, and pooler.
attention_dropout (`float`, *optional*, defaults to 0.0):
@@ -208,12 +208,11 @@ class CLIPVisionConfig(PretrainedConfig):
class CLIPConfig(PretrainedConfig):
r"""
- [`CLIPConfig`] is the configuration class to store the configuration of a
- [`CLIPModel`]. It is used to instantiate CLIP model according to the specified arguments,
- defining the text model and vision model configs.
+ [`CLIPConfig`] is the configuration class to store the configuration of a [`CLIPModel`]. It is used to instantiate
+ CLIP model according to the specified arguments, defining the text model and vision model configs.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
text_config_dict (`dict`, *optional*):
@@ -259,8 +258,8 @@ class CLIPConfig(PretrainedConfig):
@classmethod
def from_text_vision_configs(cls, text_config: CLIPTextConfig, vision_config: CLIPVisionConfig, **kwargs):
r"""
- Instantiate a [`CLIPConfig`] (or a derived class) from clip text model configuration and
- clip vision model configuration.
+ Instantiate a [`CLIPConfig`] (or a derived class) from clip text model configuration and clip vision model
+ configuration.
Returns:
[`CLIPConfig`]: An instance of a configuration object
@@ -270,8 +269,7 @@ class CLIPConfig(PretrainedConfig):
def to_dict(self):
"""
- Serializes this instance to a Python dictionary. Override the default
- [`~PretrainedConfig.to_dict`].
+ Serializes this instance to a Python dictionary. Override the default [`~PretrainedConfig.to_dict`].
Returns:
`Dict[str, any]`: Dictionary of all the attributes that make up this configuration instance,
diff --git a/src/transformers/models/clip/feature_extraction_clip.py b/src/transformers/models/clip/feature_extraction_clip.py
index 45c5289c90..a480ce4580 100644
--- a/src/transformers/models/clip/feature_extraction_clip.py
+++ b/src/transformers/models/clip/feature_extraction_clip.py
@@ -32,8 +32,8 @@ class CLIPFeatureExtractor(FeatureExtractionMixin, ImageFeatureExtractionMixin):
r"""
Constructs a CLIP feature extractor.
- This feature extractor inherits from [`FeatureExtractionMixin`] which contains most of the main
- methods. Users should refer to this superclass for more information regarding those methods.
+ This feature extractor inherits from [`FeatureExtractionMixin`] which contains most of the main methods. Users
+ should refer to this superclass for more information regarding those methods.
Args:
do_resize (`bool`, *optional*, defaults to `True`):
@@ -42,14 +42,13 @@ class CLIPFeatureExtractor(FeatureExtractionMixin, ImageFeatureExtractionMixin):
Resize the input to the given size. Only has an effect if `do_resize` is set to `True`.
resample (`int`, *optional*, defaults to `PIL.Image.BICUBIC`):
An optional resampling filter. This can be one of `PIL.Image.NEAREST`, `PIL.Image.BOX`,
- `PIL.Image.BILINEAR`, `PIL.Image.HAMMING`, `PIL.Image.BICUBIC` or `PIL.Image.LANCZOS`.
- Only has an effect if `do_resize` is set to `True`.
+ `PIL.Image.BILINEAR`, `PIL.Image.HAMMING`, `PIL.Image.BICUBIC` or `PIL.Image.LANCZOS`. Only has an effect
+ if `do_resize` is set to `True`.
do_center_crop (`bool`, *optional*, defaults to `True`):
- Whether to crop the input at the center. If the input size is smaller than `crop_size` along any edge,
- the image is padded with 0's and then center cropped.
+ Whether to crop the input at the center. If the input size is smaller than `crop_size` along any edge, the
+ image is padded with 0's and then center cropped.
crop_size (`int`, *optional*, defaults to 224):
- Desired output size when applying center-cropping. Only has an effect if `do_center_crop` is set to
- `True`.
+ Desired output size when applying center-cropping. Only has an effect if `do_center_crop` is set to `True`.
do_normalize (`bool`, *optional*, defaults to `True`):
Whether or not to normalize the input with `image_mean` and `image_std`.
image_mean (`List[int]`, defaults to `[0.485, 0.456, 0.406]`):
@@ -159,8 +158,8 @@ class CLIPFeatureExtractor(FeatureExtractionMixin, ImageFeatureExtractionMixin):
def center_crop(self, image, size):
"""
- Crops `image` to the given size using a center crop. Note that if the image is too small to be cropped to
- the size is given, it will be padded (so the returned result has the size asked).
+ Crops `image` to the given size using a center crop. Note that if the image is too small to be cropped to the
+ size is given, it will be padded (so the returned result has the size asked).
Args:
image (`PIL.Image.Image` or `np.ndarray` or `torch.Tensor`):
diff --git a/src/transformers/models/clip/modeling_clip.py b/src/transformers/models/clip/modeling_clip.py
index 45e7c0b307..aabfdf5bfc 100755
--- a/src/transformers/models/clip/modeling_clip.py
+++ b/src/transformers/models/clip/modeling_clip.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" PyTorch CLIP model. """
+""" PyTorch CLIP model."""
from dataclasses import dataclass
@@ -79,17 +79,15 @@ class CLIPOutput(ModelOutput):
loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `return_loss` is `True`):
Contrastive loss for image-text similarity.
logits_per_image:(`torch.FloatTensor` of shape `(image_batch_size, text_batch_size)`):
- The scaled dot product scores between `image_embeds` and `text_embeds`. This represents the
- image-text similarity scores.
+ The scaled dot product scores between `image_embeds` and `text_embeds`. This represents the image-text
+ similarity scores.
logits_per_text:(`torch.FloatTensor` of shape `(text_batch_size, image_batch_size)`):
- The scaled dot product scores between `text_embeds` and `image_embeds`. This represents the
- text-image similarity scores.
+ The scaled dot product scores between `text_embeds` and `image_embeds`. This represents the text-image
+ similarity scores.
text_embeds(`torch.FloatTensor` of shape `(batch_size, output_dim`):
- The text embeddings obtained by applying the projection layer to the pooled output of
- [`CLIPTextModel`].
+ The text embeddings obtained by applying the projection layer to the pooled output of [`CLIPTextModel`].
image_embeds(`torch.FloatTensor` of shape `(batch_size, output_dim`):
- The image embeddings obtained by applying the projection layer to the pooled output of
- [`CLIPVisionModel`].
+ The image embeddings obtained by applying the projection layer to the pooled output of [`CLIPVisionModel`].
text_model_output(`BaseModelOutputWithPooling`):
The output of the [`CLIPTextModel`].
vision_model_output(`BaseModelOutputWithPooling`):
@@ -391,15 +389,14 @@ class CLIPPreTrainedModel(PreTrainedModel):
CLIP_START_DOCSTRING = r"""
- This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use
- it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
+ This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it
+ as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
behavior.
Parameters:
config ([`CLIPConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
CLIP_TEXT_INPUTS_DOCSTRING = r"""
@@ -408,9 +405,8 @@ CLIP_TEXT_INPUTS_DOCSTRING = r"""
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
it.
- Indices can be obtained using [`CLIPTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`CLIPTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -421,7 +417,8 @@ CLIP_TEXT_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
output_attentions (`bool`, *optional*):
@@ -438,8 +435,7 @@ CLIP_VISION_INPUTS_DOCSTRING = r"""
Args:
pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using
- [`CLIPFeatureExtractor`]. See [`CLIPFeatureExtractor.__call__`] for
- details.
+ [`CLIPFeatureExtractor`]. See [`CLIPFeatureExtractor.__call__`] for details.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -456,9 +452,8 @@ CLIP_INPUTS_DOCSTRING = r"""
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
it.
- Indices can be obtained using [`CLIPTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`CLIPTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -469,13 +464,13 @@ CLIP_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using
- [`CLIPFeatureExtractor`]. See [`CLIPFeatureExtractor.__call__`] for
- details.
+ [`CLIPFeatureExtractor`]. See [`CLIPFeatureExtractor.__call__`] for details.
return_loss (`bool`, *optional*):
Whether or not to return the contrastive loss.
output_attentions (`bool`, *optional*):
@@ -516,9 +511,9 @@ class CLIPEncoder(nn.Module):
r"""
Args:
inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded
- representation. This is useful if you want more control over how to convert `input_ids` indices
- into associated vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
+ This is useful if you want more control over how to convert `input_ids` indices into associated vectors
+ than the model's internal embedding lookup matrix.
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
@@ -884,8 +879,8 @@ class CLIPModel(CLIPPreTrainedModel):
):
r"""
Returns:
- text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings
- obtained by applying the projection layer to the pooled output of [`CLIPTextModel`].
+ text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by
+ applying the projection layer to the pooled output of [`CLIPTextModel`].
Examples:
@@ -922,8 +917,8 @@ class CLIPModel(CLIPPreTrainedModel):
):
r"""
Returns:
- image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings
- obtained by applying the projection layer to the pooled output of [`CLIPVisionModel`].
+ image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by
+ applying the projection layer to the pooled output of [`CLIPVisionModel`].
Examples:
diff --git a/src/transformers/models/clip/modeling_flax_clip.py b/src/transformers/models/clip/modeling_flax_clip.py
index 97db507467..cbb811ab67 100644
--- a/src/transformers/models/clip/modeling_flax_clip.py
+++ b/src/transformers/models/clip/modeling_flax_clip.py
@@ -40,12 +40,12 @@ logger = logging.get_logger(__name__)
CLIP_START_DOCSTRING = r"""
- This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading, saving and converting weights from
- PyTorch models)
+ This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading, saving and converting weights from PyTorch models)
- This model is also a Flax Linen [flax.linen.Module](https://flax.readthedocs.io/en/latest/flax.linen.html#module) subclass. Use it as a regular Flax linen Module
- and refer to the Flax documentation for all matter related to general usage and behavior.
+ This model is also a Flax Linen [flax.linen.Module](https://flax.readthedocs.io/en/latest/flax.linen.html#module)
+ subclass. Use it as a regular Flax linen Module and refer to the Flax documentation for all matter related to
+ general usage and behavior.
Finally, this model supports inherent JAX features such as:
@@ -57,11 +57,10 @@ CLIP_START_DOCSTRING = r"""
Parameters:
config ([`CLIPConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the
- model weights.
+ configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the model weights.
dtype (`jax.numpy.dtype`, *optional*, defaults to `jax.numpy.float32`):
- The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on
- GPUs) and `jax.numpy.bfloat16` (on TPUs).
+ The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on GPUs) and
+ `jax.numpy.bfloat16` (on TPUs).
This can be used to enable mixed-precision training or half-precision inference on GPUs or TPUs. If
specified all the computation will be performed with the given `dtype`.
@@ -69,8 +68,8 @@ CLIP_START_DOCSTRING = r"""
**Note that this only specifies the dtype of the computation and does not influence the dtype of model
parameters.**
- If you wish to change the dtype of the model parameters, see
- [`~FlaxPreTrainedModel.to_fp16`] and [`~FlaxPreTrainedModel.to_bf16`].
+ If you wish to change the dtype of the model parameters, see [`~FlaxPreTrainedModel.to_fp16`] and
+ [`~FlaxPreTrainedModel.to_bf16`].
"""
CLIP_TEXT_INPUTS_DOCSTRING = r"""
@@ -79,9 +78,8 @@ CLIP_TEXT_INPUTS_DOCSTRING = r"""
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
it.
- Indices can be obtained using [`CLIPTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`CLIPTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`numpy.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
@@ -92,7 +90,8 @@ CLIP_TEXT_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
position_ids (`numpy.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
output_attentions (`bool`, *optional*):
@@ -109,8 +108,7 @@ CLIP_VISION_INPUTS_DOCSTRING = r"""
Args:
pixel_values (`numpy.ndarray` of shape `(batch_size, num_channels, height, width)`):
Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using
- [`CLIPFeatureExtractor`]. See [`CLIPFeatureExtractor.__call__`] for
- details.
+ [`CLIPFeatureExtractor`]. See [`CLIPFeatureExtractor.__call__`] for details.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -127,9 +125,8 @@ CLIP_INPUTS_DOCSTRING = r"""
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
it.
- Indices can be obtained using [`CLIPTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`CLIPTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`numpy.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
@@ -140,13 +137,13 @@ CLIP_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
position_ids (`numpy.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
pixel_values (`numpy.ndarray` of shape `(batch_size, num_channels, height, width)`):
Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using
- [`CLIPFeatureExtractor`]. See [`CLIPFeatureExtractor.__call__`] for
- details.
+ [`CLIPFeatureExtractor`]. See [`CLIPFeatureExtractor.__call__`] for details.
return_loss (`bool`, *optional*):
Whether or not to return the contrastive loss.
output_attentions (`bool`, *optional*):
@@ -165,11 +162,11 @@ class FlaxCLIPOutput(ModelOutput):
"""
Args:
logits_per_image:(`jnp.ndarray` of shape `(image_batch_size, text_batch_size)`):
- The scaled dot product scores between `image_embeds` and `text_embeds`. This represents the
- image-text similarity scores.
+ The scaled dot product scores between `image_embeds` and `text_embeds`. This represents the image-text
+ similarity scores.
logits_per_text:(`jnp.ndarray` of shape `(text_batch_size, image_batch_size)`):
- The scaled dot product scores between `text_embeds` and `image_embeds`. This represents the
- text-image similarity scores.
+ The scaled dot product scores between `text_embeds` and `image_embeds`. This represents the text-image
+ similarity scores.
text_embeds(`jnp.ndarray` of shape `(batch_size, output_dim`):
The text embeddings obtained by applying the projection layer to the pooled output of
[`FlaxCLIPTextModel`].
@@ -799,15 +796,14 @@ class FlaxCLIPPreTrainedModel(FlaxPreTrainedModel):
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
provide it.
- Indices can be obtained using [`CLIPTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`]
- for details.
+ Indices can be obtained using [`CLIPTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
Returns:
- text_features (`jnp.ndarray` of shape `(batch_size, output_dim`): The text embeddings obtained by
- applying the projection layer to the pooled output of [`FlaxCLIPTextModel`].
+ text_features (`jnp.ndarray` of shape `(batch_size, output_dim`): The text embeddings obtained by applying
+ the projection layer to the pooled output of [`FlaxCLIPTextModel`].
Examples:
@@ -859,12 +855,11 @@ class FlaxCLIPPreTrainedModel(FlaxPreTrainedModel):
Args:
pixel_values (`numpy.ndarray` of shape `(batch_size, num_channels, height, width)`):
Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained
- using [`CLIPFeatureExtractor`]. See
- [`CLIPFeatureExtractor.__call__`] for details.
+ using [`CLIPFeatureExtractor`]. See [`CLIPFeatureExtractor.__call__`] for details.
Returns:
- image_features (`jnp.ndarray` of shape `(batch_size, output_dim`): The image embeddings obtained
- by applying the projection layer to the pooled output of [`FlaxCLIPVisionModel`]
+ image_features (`jnp.ndarray` of shape `(batch_size, output_dim`): The image embeddings obtained by
+ applying the projection layer to the pooled output of [`FlaxCLIPVisionModel`]
Examples:
diff --git a/src/transformers/models/clip/modeling_tf_clip.py b/src/transformers/models/clip/modeling_tf_clip.py
index 918ecdd1be..a32037db21 100644
--- a/src/transformers/models/clip/modeling_tf_clip.py
+++ b/src/transformers/models/clip/modeling_tf_clip.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" TF 2.0 CLIP model. """
+""" TF 2.0 CLIP model."""
import math
@@ -95,14 +95,13 @@ class TFCLIPOutput(ModelOutput):
loss (`tf.Tensor` of shape `(1,)`, *optional*, returned when `return_loss` is `True`):
Contrastive loss for image-text similarity.
logits_per_image:(`tf.Tensor` of shape `(image_batch_size, text_batch_size)`):
- The scaled dot product scores between `image_embeds` and `text_embeds`. This represents the
- image-text similarity scores.
+ The scaled dot product scores between `image_embeds` and `text_embeds`. This represents the image-text
+ similarity scores.
logits_per_text:(`tf.Tensor` of shape `(text_batch_size, image_batch_size)`):
- The scaled dot product scores between `text_embeds` and `image_embeds`. This represents the
- text-image similarity scores.
+ The scaled dot product scores between `text_embeds` and `image_embeds`. This represents the text-image
+ similarity scores.
text_embeds(`tf.Tensor` of shape `(batch_size, output_dim`):
- The text embeddings obtained by applying the projection layer to the pooled output of
- [`TFCLIPTextModel`].
+ The text embeddings obtained by applying the projection layer to the pooled output of [`TFCLIPTextModel`].
image_embeds(`tf.Tensor` of shape `(batch_size, output_dim`):
The image embeddings obtained by applying the projection layer to the pooled output of
[`TFCLIPVisionModel`].
@@ -965,13 +964,13 @@ class TFCLIPPreTrainedModel(TFPreTrainedModel):
CLIP_START_DOCSTRING = r"""
- This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading or saving, resizing the input
- embeddings, pruning heads etc.)
+ This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use
- it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage
- and behavior.
+ This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it
+ as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and
+ behavior.
@@ -980,11 +979,11 @@ CLIP_START_DOCSTRING = r"""
- having all inputs as keyword arguments (like PyTorch models), or
- having all inputs as a list, tuple or dict in the first positional arguments.
- This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all
- the tensors in the first argument of the model call function: `model(inputs)`.
+ This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all the
+ tensors in the first argument of the model call function: `model(inputs)`.
- If you choose this second option, there are three possibilities you can use to gather all the input Tensors in
- the first positional argument :
+ If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the
+ first positional argument :
- a single Tensor with `input_ids` only and nothing else: `model(input_ids)`
- a list of varying length with one or several input Tensors IN THE ORDER given in the docstring:
@@ -997,8 +996,7 @@ CLIP_START_DOCSTRING = r"""
Args:
config ([`CLIPConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~TFPreTrainedModel.from_pretrained`] method to load the
- model weights.
+ configuration. Check out the [`~TFPreTrainedModel.from_pretrained`] method to load the model weights.
"""
CLIP_TEXT_INPUTS_DOCSTRING = r"""
@@ -1006,9 +1004,8 @@ CLIP_TEXT_INPUTS_DOCSTRING = r"""
input_ids (`np.ndarray`, `tf.Tensor`, `List[tf.Tensor]` ``Dict[str, tf.Tensor]` or `Dict[str, np.ndarray]` and each example must have the shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`BertTokenizer`]. See
- [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for
- details.
+ Indices can be obtained using [`BertTokenizer`]. See [`PreTrainedTokenizer.__call__`] and
+ [`PreTrainedTokenizer.encode`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`np.ndarray` or `tf.Tensor` of shape `({0})`, *optional*):
@@ -1019,7 +1016,8 @@ CLIP_TEXT_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
position_ids (`np.ndarray` or `tf.Tensor` of shape `({0})`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
output_attentions (`bool`, *optional*):
@@ -1031,8 +1029,8 @@ CLIP_TEXT_INPUTS_DOCSTRING = r"""
more detail. This argument can be used only in eager mode, in graph mode the value in the config will be
used instead.
return_dict (`bool`, *optional*):
- Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This
- argument can be used in eager mode, in graph mode the value will always be set to True.
+ Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This argument can be used
+ in eager mode, in graph mode the value will always be set to True.
training (`bool`, *optional*, defaults to `False``):
Whether or not to use the model in training mode (some modules like dropout modules have different
behaviors between training and evaluation).
@@ -1040,16 +1038,19 @@ CLIP_TEXT_INPUTS_DOCSTRING = r"""
CLIP_VISION_INPUTS_DOCSTRING = r"""
Args:
- pixel_values (`np.ndarray`, `tf.Tensor`, `List[tf.Tensor]` ``Dict[str, tf.Tensor]` or `Dict[str, np.ndarray]` and each example must have the shape `(batch_size, num_channels, height, width)`): Pixel values. Pixel values can be obtained using [`CLIPFeatureExtractor`]. See [`CLIPFeatureExtractor.__call__`] for details. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
- tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the
- config will be used instead.
+ pixel_values (`np.ndarray`, `tf.Tensor`, `List[tf.Tensor]` ``Dict[str, tf.Tensor]` or `Dict[str, np.ndarray]` and each example must have the shape `(batch_size, num_channels, height, width)`):
+ Pixel values. Pixel values can be obtained using [`CLIPFeatureExtractor`]. See
+ [`CLIPFeatureExtractor.__call__`] for details. output_attentions (`bool`, *optional*): Whether or not to
+ return the attentions tensors of all attention layers. See `attentions` under returned tensors for more
+ detail. This argument can be used only in eager mode, in graph mode the value in the config will be used
+ instead.
output_hidden_states (`bool`, *optional*):
Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
more detail. This argument can be used only in eager mode, in graph mode the value in the config will be
used instead.
return_dict (`bool`, *optional*):
- Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This
- argument can be used in eager mode, in graph mode the value will always be set to True.
+ Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This argument can be used
+ in eager mode, in graph mode the value will always be set to True.
training (`bool`, *optional*, defaults to `False``):
Whether or not to use the model in training mode (some modules like dropout modules have different
behaviors between training and evaluation).
@@ -1060,9 +1061,8 @@ CLIP_INPUTS_DOCSTRING = r"""
input_ids (`np.ndarray`, `tf.Tensor`, `List[tf.Tensor]` ``Dict[str, tf.Tensor]` or `Dict[str, np.ndarray]` and each example must have the shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`BertTokenizer`]. See
- [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for
- details.
+ Indices can be obtained using [`BertTokenizer`]. See [`PreTrainedTokenizer.__call__`] and
+ [`PreTrainedTokenizer.encode`] for details.
[What are input IDs?](../glossary#input-ids)
pixel_values (`np.ndarray`, `tf.Tensor`, `List[tf.Tensor]` `Dict[str, tf.Tensor]` or `Dict[str, np.ndarray]` and each example must have the shape `(batch_size, num_channels, height, width)`):
@@ -1076,7 +1076,8 @@ CLIP_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
position_ids (`np.ndarray` or `tf.Tensor` of shape `({0})`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
return_loss (`bool`, *optional*):
@@ -1090,8 +1091,8 @@ CLIP_INPUTS_DOCSTRING = r"""
more detail. This argument can be used only in eager mode, in graph mode the value in the config will be
used instead.
return_dict (`bool`, *optional*):
- Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This
- argument can be used in eager mode, in graph mode the value will always be set to True.
+ Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This argument can be used
+ in eager mode, in graph mode the value will always be set to True.
training (`bool`, *optional*, defaults to `False``):
Whether or not to use the model in training mode (some modules like dropout modules have different
behaviors between training and evaluation).
@@ -1343,8 +1344,8 @@ class TFCLIPModel(TFCLIPPreTrainedModel):
) -> tf.Tensor:
r"""
Returns:
- text_features (`tf.Tensor` of shape `(batch_size, output_dim`): The text embeddings obtained by
- applying the projection layer to the pooled output of [`TFCLIPTextModel`].
+ text_features (`tf.Tensor` of shape `(batch_size, output_dim`): The text embeddings obtained by applying
+ the projection layer to the pooled output of [`TFCLIPTextModel`].
Examples:
@@ -1393,8 +1394,8 @@ class TFCLIPModel(TFCLIPPreTrainedModel):
) -> tf.Tensor:
r"""
Returns:
- image_features (`tf.Tensor` of shape `(batch_size, output_dim`): The image embeddings obtained by
- applying the projection layer to the pooled output of [`TFCLIPVisionModel`].
+ image_features (`tf.Tensor` of shape `(batch_size, output_dim`): The image embeddings obtained by applying
+ the projection layer to the pooled output of [`TFCLIPVisionModel`].
Examples:
diff --git a/src/transformers/models/clip/processing_clip.py b/src/transformers/models/clip/processing_clip.py
index caae7983c1..fe516b1745 100644
--- a/src/transformers/models/clip/processing_clip.py
+++ b/src/transformers/models/clip/processing_clip.py
@@ -24,9 +24,8 @@ class CLIPProcessor:
r"""
Constructs a CLIP processor which wraps a CLIP feature extractor and a CLIP tokenizer into a single processor.
- [`CLIPProcessor`] offers all the functionalities of [`CLIPFeatureExtractor`]
- and [`CLIPTokenizer`]. See the [`~CLIPProcessor.__call__`] and
- [`~CLIPProcessor.decode`] for more information.
+ [`CLIPProcessor`] offers all the functionalities of [`CLIPFeatureExtractor`] and [`CLIPTokenizer`]. See the
+ [`~CLIPProcessor.__call__`] and [`~CLIPProcessor.decode`] for more information.
Args:
feature_extractor ([`CLIPFeatureExtractor`]):
@@ -55,8 +54,8 @@ class CLIPProcessor:
This class method is simply calling [`~PreTrainedFeatureExtractor.save_pretrained`] and
- [`~tokenization_utils_base.PreTrainedTokenizer.save_pretrained`]. Please refer to the
- docstrings of the methods above for more information.
+ [`~tokenization_utils_base.PreTrainedTokenizer.save_pretrained`]. Please refer to the docstrings of the methods
+ above for more information.
@@ -76,9 +75,8 @@ class CLIPProcessor:
- This class method is simply calling CLIPFeatureExtractor's
- [`~PreTrainedFeatureExtractor.from_pretrained`] and CLIPTokenizer's
- [`~tokenization_utils_base.PreTrainedTokenizer.from_pretrained`]. Please refer to the
+ This class method is simply calling CLIPFeatureExtractor's [`~PreTrainedFeatureExtractor.from_pretrained`] and
+ CLIPTokenizer's [`~tokenization_utils_base.PreTrainedTokenizer.from_pretrained`]. Please refer to the
docstrings of the methods above for more information.
@@ -91,8 +89,7 @@ class CLIPProcessor:
huggingface.co. Valid model ids can be located at the root-level, like `clip-vit-base-patch32`, or
namespaced under a user or organization name, like `openai/clip-vit-base-patch32`.
- a path to a *directory* containing a feature extractor file saved using the
- [`~PreTrainedFeatureExtractor.save_pretrained`] method, e.g.,
- `./my_model_directory/`.
+ [`~PreTrainedFeatureExtractor.save_pretrained`] method, e.g., `./my_model_directory/`.
- a path or url to a saved feature extractor JSON *file*, e.g.,
`./my_model_directory/preprocessor_config.json`.
@@ -107,11 +104,10 @@ class CLIPProcessor:
def __call__(self, text=None, images=None, return_tensors=None, **kwargs):
"""
- Main method to prepare for the model one or several sequences(s) and image(s). This method forwards the
- `text` and `kwargs` arguments to CLIPTokenizer's [`~CLIPTokenizer.__call__`] if
- `text` is not `None` to encode the text. To prepare the image(s), this method forwards the
- `images` and `kwrags` arguments to CLIPFeatureExtractor's
- [`~CLIPFeatureExtractor.__call__`] if `images` is not `None`. Please refer to the
+ Main method to prepare for the model one or several sequences(s) and image(s). This method forwards the `text`
+ and `kwargs` arguments to CLIPTokenizer's [`~CLIPTokenizer.__call__`] if `text` is not `None` to encode the
+ text. To prepare the image(s), this method forwards the `images` and `kwrags` arguments to
+ CLIPFeatureExtractor's [`~CLIPFeatureExtractor.__call__`] if `images` is not `None`. Please refer to the
doctsring of the above two methods for more information.
Args:
@@ -137,8 +133,8 @@ class CLIPProcessor:
- **input_ids** -- List of token ids to be fed to a model. Returned when `text` is not `None`.
- **attention_mask** -- List of indices specifying which tokens should be attended to by the model (when
- `return_attention_mask=True` or if *"attention_mask"* is in `self.model_input_names` and if
- `text` is not `None`).
+ `return_attention_mask=True` or if *"attention_mask"* is in `self.model_input_names` and if `text` is not
+ `None`).
- **pixel_values** -- Pixel values to be fed to a model. Returned when `images` is not `None`.
"""
@@ -161,15 +157,14 @@ class CLIPProcessor:
def batch_decode(self, *args, **kwargs):
"""
- This method forwards all its arguments to CLIPTokenizer's
- [`~PreTrainedTokenizer.batch_decode`]. Please refer to the docstring of this method for more
- information.
+ This method forwards all its arguments to CLIPTokenizer's [`~PreTrainedTokenizer.batch_decode`]. Please refer
+ to the docstring of this method for more information.
"""
return self.tokenizer.batch_decode(*args, **kwargs)
def decode(self, *args, **kwargs):
"""
- This method forwards all its arguments to CLIPTokenizer's [`~PreTrainedTokenizer.decode`].
- Please refer to the docstring of this method for more information.
+ This method forwards all its arguments to CLIPTokenizer's [`~PreTrainedTokenizer.decode`]. Please refer to the
+ docstring of this method for more information.
"""
return self.tokenizer.decode(*args, **kwargs)
diff --git a/src/transformers/models/clip/tokenization_clip.py b/src/transformers/models/clip/tokenization_clip.py
index a3da5bb56e..c81e5aa4ba 100644
--- a/src/transformers/models/clip/tokenization_clip.py
+++ b/src/transformers/models/clip/tokenization_clip.py
@@ -110,13 +110,12 @@ class CLIPTokenizer(PreTrainedTokenizer):
- When used with `is_split_into_words=True`, this tokenizer will add a space before each word (even the first
- one).
+ When used with `is_split_into_words=True`, this tokenizer will add a space before each word (even the first one).
- This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods.
- Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
+ this superclass for more information regarding those methods.
Args:
vocab_file (`str`):
@@ -124,7 +123,8 @@ class CLIPTokenizer(PreTrainedTokenizer):
merges_file (`str`):
Path to the merges file.
errors (`str`, *optional*, defaults to `"replace"`):
- Paradigm to follow when decoding bytes to UTF-8. See [bytes.decode](https://docs.python.org/3/library/stdtypes.html#bytes.decode) for more information.
+ Paradigm to follow when decoding bytes to UTF-8. See
+ [bytes.decode](https://docs.python.org/3/library/stdtypes.html#bytes.decode) for more information.
unk_token (`str`, *optional*, defaults to `<|endoftext|>`):
The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
token instead.
@@ -201,8 +201,7 @@ class CLIPTokenizer(PreTrainedTokenizer):
@property
def pad_token_id(self) -> Optional[int]:
"""
- `Optional[int]`: Id of the padding token in the vocabulary. Returns `None` if the token has not been
- set.
+ `Optional[int]`: Id of the padding token in the vocabulary. Returns `None` if the token has not been set.
"""
return 0
diff --git a/src/transformers/models/clip/tokenization_clip_fast.py b/src/transformers/models/clip/tokenization_clip_fast.py
index 1870c3b8ae..42713e4b39 100644
--- a/src/transformers/models/clip/tokenization_clip_fast.py
+++ b/src/transformers/models/clip/tokenization_clip_fast.py
@@ -69,13 +69,12 @@ class CLIPTokenizerFast(PreTrainedTokenizerFast):
- When used with `is_split_into_words=True`, this tokenizer needs to be instantiated with
- `add_prefix_space=True`.
+ When used with `is_split_into_words=True`, this tokenizer needs to be instantiated with `add_prefix_space=True`.
- This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main
- methods. Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should
+ refer to this superclass for more information regarding those methods.
Args:
vocab_file (`str`):
@@ -83,7 +82,8 @@ class CLIPTokenizerFast(PreTrainedTokenizerFast):
merges_file (`str`):
Path to the merges file.
errors (`str`, *optional*, defaults to `"replace"`):
- Paradigm to follow when decoding bytes to UTF-8. See [bytes.decode](https://docs.python.org/3/library/stdtypes.html#bytes.decode) for more information.
+ Paradigm to follow when decoding bytes to UTF-8. See
+ [bytes.decode](https://docs.python.org/3/library/stdtypes.html#bytes.decode) for more information.
unk_token (`str`, *optional*, defaults to `<|endoftext|>`):
The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
token instead.
@@ -140,8 +140,7 @@ class CLIPTokenizerFast(PreTrainedTokenizerFast):
@property
def pad_token_id(self) -> Optional[int]:
"""
- `Optional[int]`: Id of the padding token in the vocabulary. Returns `None` if the token has not been
- set.
+ `Optional[int]`: Id of the padding token in the vocabulary. Returns `None` if the token has not been set.
"""
return 0
diff --git a/src/transformers/models/convbert/configuration_convbert.py b/src/transformers/models/convbert/configuration_convbert.py
index bce7518754..8f0c6fd773 100644
--- a/src/transformers/models/convbert/configuration_convbert.py
+++ b/src/transformers/models/convbert/configuration_convbert.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" ConvBERT model configuration """
+""" ConvBERT model configuration"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging
@@ -30,17 +30,17 @@ CONVBERT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class ConvBertConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`ConvBertModel`]. It is used to
- instantiate an ConvBERT model according to the specified arguments, defining the model architecture. Instantiating
- a configuration with the defaults will yield a similar configuration to that of the ConvBERT [conv-bert-base](https://huggingface.co/YituTech/conv-bert-base) architecture. Configuration objects inherit from
- [`PretrainedConfig`] and can be used to control the model outputs. Read the documentation from
- [`PretrainedConfig`] for more information.
+ This is the configuration class to store the configuration of a [`ConvBertModel`]. It is used to instantiate an
+ ConvBERT model according to the specified arguments, defining the model architecture. Instantiating a configuration
+ with the defaults will yield a similar configuration to that of the ConvBERT
+ [conv-bert-base](https://huggingface.co/YituTech/conv-bert-base) architecture. Configuration objects inherit from
+ [`PretrainedConfig`] and can be used to control the model outputs. Read the documentation from [`PretrainedConfig`]
+ for more information.
Args:
vocab_size (`int`, *optional*, defaults to 30522):
Vocabulary size of the ConvBERT model. Defines the number of different tokens that can be represented by
- the `inputs_ids` passed when calling [`ConvBertModel`] or
- [`TFConvBertModel`].
+ the `inputs_ids` passed when calling [`ConvBertModel`] or [`TFConvBertModel`].
hidden_size (`int`, *optional*, defaults to 768):
Dimensionality of the encoder layers and the pooler layer.
num_hidden_layers (`int`, *optional*, defaults to 12):
@@ -50,8 +50,8 @@ class ConvBertConfig(PretrainedConfig):
intermediate_size (`int`, *optional*, defaults to 3072):
Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"selu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"selu"` and `"gelu_new"` are supported.
hidden_dropout_prob (`float`, *optional*, defaults to 0.1):
The dropout probabilitiy for all fully connected layers in the embeddings, encoder, and pooler.
attention_probs_dropout_prob (`float`, *optional*, defaults to 0.1):
@@ -60,8 +60,7 @@ class ConvBertConfig(PretrainedConfig):
The maximum sequence length that this model might ever be used with. Typically set this to something large
just in case (e.g., 512 or 1024 or 2048).
type_vocab_size (`int`, *optional*, defaults to 2):
- The vocabulary size of the `token_type_ids` passed when calling [`ConvBertModel`]
- or [`TFConvBertModel`].
+ The vocabulary size of the `token_type_ids` passed when calling [`ConvBertModel`] or [`TFConvBertModel`].
initializer_range (`float`, *optional*, defaults to 0.02):
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
layer_norm_eps (`float`, *optional*, defaults to 1e-12):
diff --git a/src/transformers/models/convbert/modeling_convbert.py b/src/transformers/models/convbert/modeling_convbert.py
index 51018e648e..4677176e77 100755
--- a/src/transformers/models/convbert/modeling_convbert.py
+++ b/src/transformers/models/convbert/modeling_convbert.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" PyTorch ConvBERT model. """
+""" PyTorch ConvBERT model."""
import math
@@ -703,8 +703,7 @@ CONVBERT_START_DOCSTRING = r"""
Parameters:
config ([`ConvBertConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
CONVBERT_INPUTS_DOCSTRING = r"""
@@ -712,9 +711,8 @@ CONVBERT_INPUTS_DOCSTRING = r"""
input_ids (`torch.LongTensor` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`ConvBertTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`ConvBertTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
@@ -726,7 +724,8 @@ CONVBERT_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
@@ -734,7 +733,8 @@ CONVBERT_INPUTS_DOCSTRING = r"""
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
@@ -745,9 +745,9 @@ CONVBERT_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert *input_ids* indices into associated vectors
- than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -876,7 +876,7 @@ class ConvBertGeneratorPredictions(nn.Module):
return hidden_states
-@add_start_docstrings("""ConvBERT Model with a `language modeling` head on top. """, CONVBERT_START_DOCSTRING)
+@add_start_docstrings("""ConvBERT Model with a `language modeling` head on top.""", CONVBERT_START_DOCSTRING)
class ConvBertForMaskedLM(ConvBertPreTrainedModel):
def __init__(self, config):
super().__init__(config)
@@ -916,8 +916,9 @@ class ConvBertForMaskedLM(ConvBertPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1019,8 +1020,9 @@ class ConvBertForSequenceClassification(ConvBertPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1116,7 +1118,8 @@ class ConvBertForMultipleChoice(ConvBertPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
+ Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
+ num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
`input_ids` above)
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1300,12 +1303,12 @@ class ConvBertForQuestionAnswering(ConvBertPreTrainedModel):
r"""
start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the start of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the end of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
diff --git a/src/transformers/models/convbert/modeling_tf_convbert.py b/src/transformers/models/convbert/modeling_tf_convbert.py
index bb5c2b99c1..79039fe4d3 100644
--- a/src/transformers/models/convbert/modeling_tf_convbert.py
+++ b/src/transformers/models/convbert/modeling_tf_convbert.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" TF 2.0 ConvBERT model. """
+""" TF 2.0 ConvBERT model."""
import tensorflow as tf
@@ -649,13 +649,13 @@ class TFConvBertPreTrainedModel(TFPreTrainedModel):
CONVBERT_START_DOCSTRING = r"""
- This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading or saving, resizing the input
- embeddings, pruning heads etc.)
+ This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use
- it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage
- and behavior.
+ This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it
+ as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and
+ behavior.
@@ -664,11 +664,11 @@ CONVBERT_START_DOCSTRING = r"""
- having all inputs as keyword arguments (like PyTorch models), or
- having all inputs as a list, tuple or dict in the first positional arguments.
- This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all
- the tensors in the first argument of the model call function: `model(inputs)`.
+ This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all the
+ tensors in the first argument of the model call function: `model(inputs)`.
- If you choose this second option, there are three possibilities you can use to gather all the input Tensors in
- the first positional argument :
+ If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the
+ first positional argument :
- a single Tensor with `input_ids` only and nothing else: `model(inputs_ids)`
- a list of varying length with one or several input Tensors IN THE ORDER given in the docstring:
@@ -681,8 +681,7 @@ CONVBERT_START_DOCSTRING = r"""
Args:
config ([`ConvBertConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
CONVBERT_INPUTS_DOCSTRING = r"""
@@ -690,9 +689,8 @@ CONVBERT_INPUTS_DOCSTRING = r"""
input_ids (`Numpy array` or `tf.Tensor` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`ConvBertTokenizer`]. See
- [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for
- details.
+ Indices can be obtained using [`ConvBertTokenizer`]. See [`PreTrainedTokenizer.__call__`] and
+ [`PreTrainedTokenizer.encode`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`Numpy array` or `tf.Tensor` of shape `({0})`, *optional*):
@@ -703,14 +701,16 @@ CONVBERT_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`Numpy array` or `tf.Tensor` of shape `({0})`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`Numpy array` or `tf.Tensor` of shape `({0})`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
head_mask (`Numpy array` or `tf.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
@@ -720,9 +720,9 @@ CONVBERT_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
inputs_embeds (`tf.Tensor` of shape `({0}, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the
@@ -732,8 +732,8 @@ CONVBERT_INPUTS_DOCSTRING = r"""
more detail. This argument can be used only in eager mode, in graph mode the value in the config will be
used instead.
return_dict (`bool`, *optional*):
- Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This
- argument can be used in eager mode, in graph mode the value will always be set to True.
+ Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This argument can be used
+ in eager mode, in graph mode the value will always be set to True.
training (`bool`, *optional*, defaults to `False`):
Whether or not to use the model in training mode (some modules like dropout modules have different
behaviors between training and evaluation).
@@ -860,7 +860,7 @@ class TFConvBertGeneratorPredictions(tf.keras.layers.Layer):
return hidden_states
-@add_start_docstrings("""ConvBERT Model with a `language modeling` head on top. """, CONVBERT_START_DOCSTRING)
+@add_start_docstrings("""ConvBERT Model with a `language modeling` head on top.""", CONVBERT_START_DOCSTRING)
class TFConvBertForMaskedLM(TFConvBertPreTrainedModel, TFMaskedLanguageModelingLoss):
def __init__(self, config, *inputs, **kwargs):
super().__init__(config, **kwargs)
@@ -906,8 +906,9 @@ class TFConvBertForMaskedLM(TFConvBertPreTrainedModel, TFMaskedLanguageModelingL
):
r"""
labels (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
"""
inputs = input_processing(
func=self.call,
@@ -1029,8 +1030,9 @@ class TFConvBertForSequenceClassification(TFConvBertPreTrainedModel, TFSequenceC
):
r"""
labels (`tf.Tensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
inputs = input_processing(
func=self.call,
@@ -1137,8 +1139,8 @@ class TFConvBertForMultipleChoice(TFConvBertPreTrainedModel, TFMultipleChoiceLos
):
r"""
labels (`tf.Tensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices]` where `num_choices` is the size of the second dimension of the input tensors. (See
- `input_ids` above)
+ Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices]`
+ where `num_choices` is the size of the second dimension of the input tensors. (See `input_ids` above)
"""
inputs = input_processing(
func=self.call,
@@ -1370,12 +1372,12 @@ class TFConvBertForQuestionAnswering(TFConvBertPreTrainedModel, TFQuestionAnswer
r"""
start_positions (`tf.Tensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the start of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
end_positions (`tf.Tensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the end of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
"""
inputs = input_processing(
func=self.call,
diff --git a/src/transformers/models/convbert/tokenization_convbert.py b/src/transformers/models/convbert/tokenization_convbert.py
index e4a73f5018..a49e32ec00 100644
--- a/src/transformers/models/convbert/tokenization_convbert.py
+++ b/src/transformers/models/convbert/tokenization_convbert.py
@@ -45,9 +45,9 @@ PRETRAINED_INIT_CONFIGURATION = {
class ConvBertTokenizer(BertTokenizer):
r"""
- Construct a ConvBERT tokenizer. [`ConvBertTokenizer`] is identical to
- [`BertTokenizer`] and runs end-to-end tokenization: punctuation splitting and wordpiece. Refer
- to superclass [`BertTokenizer`] for usage examples and documentation concerning parameters.
+ Construct a ConvBERT tokenizer. [`ConvBertTokenizer`] is identical to [`BertTokenizer`] and runs end-to-end
+ tokenization: punctuation splitting and wordpiece. Refer to superclass [`BertTokenizer`] for usage examples and
+ documentation concerning parameters.
"""
vocab_files_names = VOCAB_FILES_NAMES
diff --git a/src/transformers/models/convbert/tokenization_convbert_fast.py b/src/transformers/models/convbert/tokenization_convbert_fast.py
index 8a0f42880c..525e369c4b 100644
--- a/src/transformers/models/convbert/tokenization_convbert_fast.py
+++ b/src/transformers/models/convbert/tokenization_convbert_fast.py
@@ -48,11 +48,10 @@ class ConvBertTokenizerFast(BertTokenizerFast):
r"""
Construct a "fast" ConvBERT tokenizer (backed by HuggingFace's *tokenizers* library).
- [`ConvBertTokenizerFast`] is identical to [`BertTokenizerFast`] and runs
- end-to-end tokenization: punctuation splitting and wordpiece.
+ [`ConvBertTokenizerFast`] is identical to [`BertTokenizerFast`] and runs end-to-end tokenization: punctuation
+ splitting and wordpiece.
- Refer to superclass [`BertTokenizerFast`] for usage examples and documentation concerning
- parameters.
+ Refer to superclass [`BertTokenizerFast`] for usage examples and documentation concerning parameters.
"""
vocab_files_names = VOCAB_FILES_NAMES
pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
diff --git a/src/transformers/models/cpm/tokenization_cpm.py b/src/transformers/models/cpm/tokenization_cpm.py
index 92262f23d5..812e887bc2 100644
--- a/src/transformers/models/cpm/tokenization_cpm.py
+++ b/src/transformers/models/cpm/tokenization_cpm.py
@@ -33,10 +33,11 @@ class CpmTokenizer(XLNetTokenizer):
def __init__(self, *args, **kwargs):
"""
- Construct a CPM tokenizer. Based on [Jieba](https://pypi.org/project/jieba/) and [SentencePiece](https://github.com/google/sentencepiece).
+ Construct a CPM tokenizer. Based on [Jieba](https://pypi.org/project/jieba/) and
+ [SentencePiece](https://github.com/google/sentencepiece).
- This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main
- methods. Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should
+ refer to this superclass for more information regarding those methods.
Args:
vocab_file (`str`):
@@ -54,8 +55,8 @@ class CpmTokenizer(XLNetTokenizer):
- When building a sequence using special tokens, this is not the token that is used for the beginning
- of sequence. The token used is the `cls_token`.
+ When building a sequence using special tokens, this is not the token that is used for the beginning of
+ sequence. The token used is the `cls_token`.
diff --git a/src/transformers/models/cpm/tokenization_cpm_fast.py b/src/transformers/models/cpm/tokenization_cpm_fast.py
index f166d6375e..748046b8d1 100644
--- a/src/transformers/models/cpm/tokenization_cpm_fast.py
+++ b/src/transformers/models/cpm/tokenization_cpm_fast.py
@@ -36,10 +36,11 @@ class CpmTokenizerFast(XLNetTokenizerFast):
def __init__(self, *args, **kwargs):
"""
- Construct a CPM tokenizer. Based on [Jieba](https://pypi.org/project/jieba/) and [SentencePiece](https://github.com/google/sentencepiece).
+ Construct a CPM tokenizer. Based on [Jieba](https://pypi.org/project/jieba/) and
+ [SentencePiece](https://github.com/google/sentencepiece).
- This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main
- methods. Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should
+ refer to this superclass for more information regarding those methods.
Args:
vocab_file (`str`):
@@ -57,8 +58,8 @@ class CpmTokenizerFast(XLNetTokenizerFast):
- When building a sequence using special tokens, this is not the token that is used for the beginning
- of sequence. The token used is the `cls_token`.
+ When building a sequence using special tokens, this is not the token that is used for the beginning of
+ sequence. The token used is the `cls_token`.
diff --git a/src/transformers/models/ctrl/configuration_ctrl.py b/src/transformers/models/ctrl/configuration_ctrl.py
index 5c8aa366a0..4e157d6cf7 100644
--- a/src/transformers/models/ctrl/configuration_ctrl.py
+++ b/src/transformers/models/ctrl/configuration_ctrl.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" Salesforce CTRL configuration """
+""" Salesforce CTRL configuration"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging
@@ -25,19 +25,18 @@ CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP = {"ctrl": "https://huggingface.co/ctrl/resol
class CTRLConfig(PretrainedConfig):
"""
- This is the configuration class to store the configuration of a [`CTRLModel`] or a
- [`TFCTRLModel`]. It is used to instantiate a CTRL model according to the specified arguments,
- defining the model architecture. Instantiating a configuration with the defaults will yield a similar configuration
- to that of the [ctrl](https://huggingface.co/ctrl) architecture from SalesForce.
+ This is the configuration class to store the configuration of a [`CTRLModel`] or a [`TFCTRLModel`]. It is used to
+ instantiate a CTRL model according to the specified arguments, defining the model architecture. Instantiating a
+ configuration with the defaults will yield a similar configuration to that of the
+ [ctrl](https://huggingface.co/ctrl) architecture from SalesForce.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
vocab_size (`int`, *optional*, defaults to 246534):
Vocabulary size of the CTRL model. Defines the number of different tokens that can be represented by the
- `inputs_ids` passed when calling [`CTRLModel`] or
- [`TFCTRLModel`].
+ `inputs_ids` passed when calling [`CTRLModel`] or [`TFCTRLModel`].
n_positions (`int`, *optional*, defaults to 256):
The maximum sequence length that this model might ever be used with. Typically set this to something large
just in case (e.g., 512 or 1024 or 2048).
diff --git a/src/transformers/models/ctrl/modeling_ctrl.py b/src/transformers/models/ctrl/modeling_ctrl.py
index 28ac883d42..97d73d8162 100644
--- a/src/transformers/models/ctrl/modeling_ctrl.py
+++ b/src/transformers/models/ctrl/modeling_ctrl.py
@@ -239,40 +239,37 @@ class CTRLPreTrainedModel(PreTrainedModel):
CTRL_START_DOCSTRING = r"""
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
Parameters:
config ([`CTRLConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
CTRL_INPUTS_DOCSTRING = r"""
Args:
input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
- `input_ids_length` = `sequence_length` if `past_key_values` is `None` else
- `past_key_values[0].shape[-2]` (`sequence_length` of input past key value states). Indices of input
- sequence tokens in the vocabulary.
+ `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values[0].shape[-2]`
+ (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary.
- If `past_key_values` is used, only input IDs that do not have their past calculated should be passed
- as `input_ids`.
+ If `past_key_values` is used, only input IDs that do not have their past calculated should be passed as
+ `input_ids`.
- Indices can be obtained using [`CTRLTokenizer`]. See
- [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for
- details.
+ Indices can be obtained using [`CTRLTokenizer`]. See [`PreTrainedTokenizer.__call__`] and
+ [`PreTrainedTokenizer.encode`] for details.
[What are input IDs?](../glossary#input-ids)
past_key_values (`Tuple[Tuple[torch.FloatTensor]]` of length `config.n_layers`):
Contains pre-computed hidden-states (key and values in the attention blocks) as computed by the model (see
- `past_key_values` output below). Can be used to speed up sequential decoding. The `input_ids` which
- have their past given to this model should not be passed as input ids as they have already been computed.
+ `past_key_values` output below). Can be used to speed up sequential decoding. The `input_ids` which have
+ their past given to this model should not be passed as input ids as they have already been computed.
attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
@@ -281,14 +278,16 @@ CTRL_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
@@ -298,12 +297,12 @@ CTRL_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
use_cache (`bool`, *optional*):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`).
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`).
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -539,8 +538,8 @@ class CTRLLMHeadModel(CTRLPreTrainedModel):
r"""
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
- `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to
- `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
+ `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
+ are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -586,9 +585,9 @@ class CTRLLMHeadModel(CTRLPreTrainedModel):
@staticmethod
def _reorder_cache(past: Tuple[Tuple[torch.Tensor]], beam_idx: torch.Tensor) -> Tuple[Tuple[torch.Tensor]]:
"""
- This function is used to re-order the `past_key_values` cache if
- [`~PreTrainedModel.beam_search`] or [`~PreTrainedModel.beam_sample`] is
- called. This is required to match `past_key_values` with the correct beam_idx at every generation step.
+ This function is used to re-order the `past_key_values` cache if [`~PreTrainedModel.beam_search`] or
+ [`~PreTrainedModel.beam_sample`] is called. This is required to match `past_key_values` with the correct
+ beam_idx at every generation step.
"""
return tuple(
tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past)
@@ -599,12 +598,12 @@ class CTRLLMHeadModel(CTRLPreTrainedModel):
@add_start_docstrings(
"""
The CTRL Model transformer with a sequence classification head on top (linear layer).
- [`CTRLForSequenceClassification`] uses the last token in order to do the classification, as
- other causal models (e.g. GPT-2) do. Since it does classification on the last token, it requires to know the
- position of the last token. If a `pad_token_id` is defined in the configuration, it finds the last token that
- is not a padding token in each row. If no `pad_token_id` is defined, it simply takes the last value in each
- row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of
- `input_ids`, it does the same (take the last value in each row of the batch).
+ [`CTRLForSequenceClassification`] uses the last token in order to do the classification, as other causal models
+ (e.g. GPT-2) do. Since it does classification on the last token, it requires to know the position of the last
+ token. If a `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in
+ each row. If no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot
+ guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last
+ value in each row of the batch).
""",
CTRL_START_DOCSTRING,
)
@@ -642,8 +641,9 @@ class CTRLForSequenceClassification(CTRLPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
diff --git a/src/transformers/models/ctrl/modeling_tf_ctrl.py b/src/transformers/models/ctrl/modeling_tf_ctrl.py
index 1d30e7beab..bb9c601e86 100644
--- a/src/transformers/models/ctrl/modeling_tf_ctrl.py
+++ b/src/transformers/models/ctrl/modeling_tf_ctrl.py
@@ -428,13 +428,13 @@ class TFCTRLPreTrainedModel(TFPreTrainedModel):
CTRL_START_DOCSTRING = r"""
- This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading or saving, resizing the input
- embeddings, pruning heads etc.)
+ This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use
- it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage
- and behavior.
+ This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it
+ as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and
+ behavior.
@@ -443,11 +443,11 @@ CTRL_START_DOCSTRING = r"""
- having all inputs as keyword arguments (like PyTorch models), or
- having all inputs as a list, tuple or dict in the first positional arguments.
- This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all
- the tensors in the first argument of the model call function: `model(inputs)`.
+ This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all the
+ tensors in the first argument of the model call function: `model(inputs)`.
- If you choose this second option, there are three possibilities you can use to gather all the input Tensors in
- the first positional argument :
+ If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the
+ first positional argument :
- a single Tensor with `input_ids` only and nothing else: `model(inputs_ids)`
- a list of varying length with one or several input Tensors IN THE ORDER given in the docstring:
@@ -460,24 +460,21 @@ CTRL_START_DOCSTRING = r"""
Parameters:
config ([`CTRLConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
CTRL_INPUTS_DOCSTRING = r"""
Args:
input_ids (`Numpy array` or `tf.Tensor` of shape `(batch_size, input_ids_length)`):
- `input_ids_length` = `sequence_length` if `past` is `None` else `past[0].shape[-2]`
- (`sequence_length` of input past key value states).
+ `input_ids_length` = `sequence_length` if `past` is `None` else `past[0].shape[-2]` (`sequence_length` of
+ input past key value states).
Indices of input sequence tokens in the vocabulary.
- If `past` is used, only input IDs that do not have their past calculated should be passed as
- `input_ids`.
+ If `past` is used, only input IDs that do not have their past calculated should be passed as `input_ids`.
- Indices can be obtained using [`CTRLTokenizer`]. See
- [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for
- details.
+ Indices can be obtained using [`CTRLTokenizer`]. See [`PreTrainedTokenizer.__call__`] and
+ [`PreTrainedTokenizer.encode`] for details.
[What are input IDs?](../glossary#input-ids)
past (`List[tf.Tensor]` of length `config.n_layers`):
@@ -492,14 +489,16 @@ CTRL_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`tf.Tensor` or `Numpy array` of shape `(batch_size, sequence_length)`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`tf.Tensor` or `Numpy array` of shape `(batch_size, sequence_length)`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
@@ -509,12 +508,11 @@ CTRL_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
inputs_embeds (`tf.Tensor` or `Numpy array` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
use_cache (`bool`, *optional*):
- If set to `True`, `past` key value states are returned and can be used to speed up decoding (see
- `past`).
+ If set to `True`, `past` key value states are returned and can be used to speed up decoding (see `past`).
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the
@@ -524,8 +522,8 @@ CTRL_INPUTS_DOCSTRING = r"""
more detail. This argument can be used only in eager mode, in graph mode the value in the config will be
used instead.
return_dict (`bool`, *optional*):
- Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This
- argument can be used in eager mode, in graph mode the value will always be set to True.
+ Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This argument can be used
+ in eager mode, in graph mode the value will always be set to True.
training (`bool`, *optional*, defaults to `False`):
Whether or not to use the model in training mode (some modules like dropout modules have different
behaviors between training and evaluation).
@@ -694,7 +692,8 @@ class TFCTRLLMHeadModel(TFCTRLPreTrainedModel, TFCausalLanguageModelingLoss):
):
r"""
labels (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the cross entropy classification loss. Indices should be in `[0, ..., config.vocab_size - 1]`.
+ Labels for computing the cross entropy classification loss. Indices should be in `[0, ...,
+ config.vocab_size - 1]`.
"""
inputs = input_processing(
func=self.call,
@@ -764,14 +763,14 @@ class TFCTRLLMHeadModel(TFCTRLPreTrainedModel, TFCausalLanguageModelingLoss):
"""
The CTRL Model transformer with a sequence classification head on top (linear layer).
- [`TFCTRLForSequenceClassification`] uses the last token in order to do the classification, as
- other causal models (e.g. GPT-1, GPT-2) do.
+ [`TFCTRLForSequenceClassification`] uses the last token in order to do the classification, as other causal models
+ (e.g. GPT-1, GPT-2) do.
Since it does classification on the last token, it requires to know the position of the last token. If a
- `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each
- row. If no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot
- guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take
- the last value in each row of the batch).
+ `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
+ no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
+ padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
+ each row of the batch).
""",
CTRL_START_DOCSTRING,
)
@@ -816,7 +815,8 @@ class TFCTRLForSequenceClassification(TFCTRLPreTrainedModel, TFSequenceClassific
):
r"""
labels (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the cross entropy classification loss. Indices should be in `[0, ..., config.vocab_size - 1]`.
+ Labels for computing the cross entropy classification loss. Indices should be in `[0, ...,
+ config.vocab_size - 1]`.
"""
inputs = input_processing(
func=self.call,
diff --git a/src/transformers/models/ctrl/tokenization_ctrl.py b/src/transformers/models/ctrl/tokenization_ctrl.py
index 86c24f3125..c44b1d329f 100644
--- a/src/transformers/models/ctrl/tokenization_ctrl.py
+++ b/src/transformers/models/ctrl/tokenization_ctrl.py
@@ -120,8 +120,8 @@ class CTRLTokenizer(PreTrainedTokenizer):
"""
Construct a CTRL tokenizer. Based on Byte-Pair-Encoding.
- This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods.
- Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
+ this superclass for more information regarding those methods.
Args:
vocab_file (`str`):
diff --git a/src/transformers/models/deberta/configuration_deberta.py b/src/transformers/models/deberta/configuration_deberta.py
index 6032e3acc3..3a70a2460d 100644
--- a/src/transformers/models/deberta/configuration_deberta.py
+++ b/src/transformers/models/deberta/configuration_deberta.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" DeBERTa model configuration """
+""" DeBERTa model configuration"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging
@@ -32,20 +32,18 @@ DEBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class DebertaConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`DebertaModel`] or a
- [`TFDebertaModel`]. It is used to instantiate a DeBERTa model according to the specified
- arguments, defining the model architecture. Instantiating a configuration with the defaults will yield a similar
- configuration to that of the DeBERTa [microsoft/deberta-base](https://huggingface.co/microsoft/deberta-base)
- architecture.
+ This is the configuration class to store the configuration of a [`DebertaModel`] or a [`TFDebertaModel`]. It is
+ used to instantiate a DeBERTa model according to the specified arguments, defining the model architecture.
+ Instantiating a configuration with the defaults will yield a similar configuration to that of the DeBERTa
+ [microsoft/deberta-base](https://huggingface.co/microsoft/deberta-base) architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Arguments:
vocab_size (`int`, *optional*, defaults to 30522):
Vocabulary size of the DeBERTa model. Defines the number of different tokens that can be represented by the
- `inputs_ids` passed when calling [`DebertaModel`] or
- [`TFDebertaModel`].
+ `inputs_ids` passed when calling [`DebertaModel`] or [`TFDebertaModel`].
hidden_size (`int`, *optional*, defaults to 768):
Dimensionality of the encoder layers and the pooler layer.
num_hidden_layers (`int`, *optional*, defaults to 12):
@@ -55,9 +53,9 @@ class DebertaConfig(PretrainedConfig):
intermediate_size (`int`, *optional*, defaults to 3072):
Dimensionality of the "intermediate" (often named feed-forward) layer in the Transformer encoder.
hidden_act (`str` or `Callable`, *optional*, defaults to `"gelu"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"silu"`, `"gelu"`, `"tanh"`, `"gelu_fast"`,
- `"mish"`, `"linear"`, `"sigmoid"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"silu"`, `"gelu"`, `"tanh"`, `"gelu_fast"`, `"mish"`, `"linear"`, `"sigmoid"` and `"gelu_new"`
+ are supported.
hidden_dropout_prob (`float`, *optional*, defaults to 0.1):
The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
attention_probs_dropout_prob (`float`, *optional*, defaults to 0.1):
@@ -66,8 +64,7 @@ class DebertaConfig(PretrainedConfig):
The maximum sequence length that this model might ever be used with. Typically set this to something large
just in case (e.g., 512 or 1024 or 2048).
type_vocab_size (`int`, *optional*, defaults to 2):
- The vocabulary size of the `token_type_ids` passed when calling [`DebertaModel`] or
- [`TFDebertaModel`].
+ The vocabulary size of the `token_type_ids` passed when calling [`DebertaModel`] or [`TFDebertaModel`].
initializer_range (`float`, *optional*, defaults to 0.02):
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
layer_norm_eps (`float`, *optional*, defaults to 1e-12):
@@ -75,15 +72,15 @@ class DebertaConfig(PretrainedConfig):
relative_attention (`bool`, *optional*, defaults to `False`):
Whether use relative position encoding.
max_relative_positions (`int`, *optional*, defaults to 1):
- The range of relative positions `[-max_position_embeddings, max_position_embeddings]`. Use the same
- value as `max_position_embeddings`.
+ The range of relative positions `[-max_position_embeddings, max_position_embeddings]`. Use the same value
+ as `max_position_embeddings`.
pad_token_id (`int`, *optional*, defaults to 0):
The value used to pad input_ids.
position_biased_input (`bool`, *optional*, defaults to `True`):
Whether add absolute position embedding to content embedding.
pos_att_type (`List[str]`, *optional*):
- The type of relative position attention, it can be a combination of `["p2c", "c2p"]`, e.g.
- `["p2c"]`, `["p2c", "c2p"]`.
+ The type of relative position attention, it can be a combination of `["p2c", "c2p"]`, e.g. `["p2c"]`,
+ `["p2c", "c2p"]`.
layer_norm_eps (`float`, optional, defaults to 1e-12):
The epsilon used by the layer normalization layers.
"""
diff --git a/src/transformers/models/deberta/modeling_deberta.py b/src/transformers/models/deberta/modeling_deberta.py
index 00496b0df5..1f179ed5df 100644
--- a/src/transformers/models/deberta/modeling_deberta.py
+++ b/src/transformers/models/deberta/modeling_deberta.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" PyTorch DeBERTa model. """
+""" PyTorch DeBERTa model."""
import math
from collections.abc import Sequence
@@ -79,7 +79,8 @@ class XSoftmax(torch.autograd.Function):
Args:
input (`torch.tensor`): The input tensor that will apply softmax.
- mask (`torch.IntTensor`): The mask matrix where 0 indicate that element will be ignored in the softmax calculation.
+ mask (`torch.IntTensor`):
+ The mask matrix where 0 indicate that element will be ignored in the softmax calculation.
dim (int): The dimension that will apply softmax
Example:
@@ -480,8 +481,8 @@ def build_relative_position(query_size, key_size, device):
Build relative position according to the query and key
We assume the absolute position of query \\(P_q\\) is range from (0, query_size) and the absolute position of key
- \\(P_k\\) is range from (0, key_size), The relative positions from query to key is \\(R_{q \\rightarrow k} =
- P_q - P_k\\)
+ \\(P_k\\) is range from (0, key_size), The relative positions from query to key is \\(R_{q \\rightarrow k} = P_q -
+ P_k\\)
Args:
query_size (int): the length of query
@@ -814,20 +815,20 @@ class DebertaPreTrainedModel(PreTrainedModel):
DEBERTA_START_DOCSTRING = r"""
- The DeBERTa model was proposed in [DeBERTa: Decoding-enhanced BERT with Disentangled Attention](https://arxiv.org/abs/2006.03654) by Pengcheng He, Xiaodong Liu, Jianfeng Gao, Weizhu Chen. It's build on top of
- BERT/RoBERTa with two improvements, i.e. disentangled attention and enhanced mask decoder. With those two
+ The DeBERTa model was proposed in [DeBERTa: Decoding-enhanced BERT with Disentangled
+ Attention](https://arxiv.org/abs/2006.03654) by Pengcheng He, Xiaodong Liu, Jianfeng Gao, Weizhu Chen. It's build
+ on top of BERT/RoBERTa with two improvements, i.e. disentangled attention and enhanced mask decoder. With those two
improvements, it out perform BERT/RoBERTa on a majority of tasks with 80GB pretraining data.
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.```
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.```
Parameters:
config ([`DebertaConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
DEBERTA_INPUTS_DOCSTRING = r"""
@@ -835,9 +836,8 @@ DEBERTA_INPUTS_DOCSTRING = r"""
input_ids (`torch.LongTensor` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`DebertaTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`DebertaTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
@@ -848,20 +848,22 @@ DEBERTA_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert *input_ids* indices into associated vectors
- than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -988,7 +990,7 @@ class DebertaModel(DebertaPreTrainedModel):
)
-@add_start_docstrings("""DeBERTa Model with a `language modeling` head on top. """, DEBERTA_START_DOCSTRING)
+@add_start_docstrings("""DeBERTa Model with a `language modeling` head on top.""", DEBERTA_START_DOCSTRING)
class DebertaForMaskedLM(DebertaPreTrainedModel):
_keys_to_ignore_on_load_unexpected = [r"pooler"]
_keys_to_ignore_on_load_missing = [r"position_ids", r"predictions.decoder.bias"]
@@ -1029,8 +1031,9 @@ class DebertaForMaskedLM(DebertaPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1169,8 +1172,9 @@ class DebertaForSequenceClassification(DebertaPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1362,12 +1366,12 @@ class DebertaForQuestionAnswering(DebertaPreTrainedModel):
r"""
start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the start of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the end of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
diff --git a/src/transformers/models/deberta/modeling_tf_deberta.py b/src/transformers/models/deberta/modeling_tf_deberta.py
index 90a539a466..308dc85d03 100644
--- a/src/transformers/models/deberta/modeling_tf_deberta.py
+++ b/src/transformers/models/deberta/modeling_tf_deberta.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" TF 2.0 DeBERTa model. """
+""" TF 2.0 DeBERTa model."""
import math
@@ -395,8 +395,8 @@ def build_relative_position(query_size, key_size):
Build relative position according to the query and key
We assume the absolute position of query \\(P_q\\) is range from (0, query_size) and the absolute position of key
- \\(P_k\\) is range from (0, key_size), The relative positions from query to key is \\(R_{q \\rightarrow k} =
- P_q - P_k\\)
+ \\(P_k\\) is range from (0, key_size), The relative positions from query to key is \\(R_{q \\rightarrow k} = P_q -
+ P_k\\)
Args:
query_size (int): the length of query
@@ -1001,13 +1001,14 @@ class TFDebertaPreTrainedModel(TFPreTrainedModel):
DEBERTA_START_DOCSTRING = r"""
- The DeBERTa model was proposed in [DeBERTa: Decoding-enhanced BERT with Disentangled Attention](https://arxiv.org/abs/2006.03654) by Pengcheng He, Xiaodong Liu, Jianfeng Gao, Weizhu Chen. It's build on top of
- BERT/RoBERTa with two improvements, i.e. disentangled attention and enhanced mask decoder. With those two
+ The DeBERTa model was proposed in [DeBERTa: Decoding-enhanced BERT with Disentangled
+ Attention](https://arxiv.org/abs/2006.03654) by Pengcheng He, Xiaodong Liu, Jianfeng Gao, Weizhu Chen. It's build
+ on top of BERT/RoBERTa with two improvements, i.e. disentangled attention and enhanced mask decoder. With those two
improvements, it out perform BERT/RoBERTa on a majority of tasks with 80GB pretraining data.
- This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use
- it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage
- and behavior.
+ This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it
+ as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and
+ behavior.
@@ -1016,11 +1017,11 @@ DEBERTA_START_DOCSTRING = r"""
- having all inputs as keyword arguments (like PyTorch models), or
- having all inputs as a list, tuple or dict in the first positional arguments.
- This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all
- the tensors in the first argument of the model call function: `model(inputs)`.
+ This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all the
+ tensors in the first argument of the model call function: `model(inputs)`.
- If you choose this second option, there are three possibilities you can use to gather all the input Tensors in
- the first positional argument :
+ If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the
+ first positional argument :
- a single Tensor with `input_ids` only and nothing else: `model(inputs_ids)`
- a list of varying length with one or several input Tensors IN THE ORDER given in the docstring:
@@ -1033,18 +1034,16 @@ DEBERTA_START_DOCSTRING = r"""
Parameters:
config ([`DebertaConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
DEBERTA_INPUTS_DOCSTRING = r"""
Args:
- input_ids (`np.ndarray`, `tf.Tensor`, `List[tf.Tensor]` ``Dict[str, tf.Tensor]` or `Dict[str, np.ndarray]` and each example must have the shape `({0})`)
+ input_ids (`np.ndarray`, `tf.Tensor`, `List[tf.Tensor]` ``Dict[str, tf.Tensor]` or `Dict[str, np.ndarray]` and each example must have the shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`DebertaTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`DebertaTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`np.ndarray` or `tf.Tensor` of shape `({0})`, *optional*):
@@ -1055,20 +1054,22 @@ DEBERTA_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`np.ndarray` or `tf.Tensor` of shape `({0})`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`np.ndarray` or `tf.Tensor` of shape `({0})`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
inputs_embeds (`np.ndarray` or `tf.Tensor` of shape `({0}, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert *input_ids* indices into associated vectors
- than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -1145,7 +1146,7 @@ class TFDebertaModel(TFDebertaPreTrainedModel):
return TFBaseModelOutput(last_hidden_state=output.last_hidden_state, hidden_states=hs, attentions=attns)
-@add_start_docstrings("""DeBERTa Model with a `language modeling` head on top. """, DEBERTA_START_DOCSTRING)
+@add_start_docstrings("""DeBERTa Model with a `language modeling` head on top.""", DEBERTA_START_DOCSTRING)
class TFDebertaForMaskedLM(TFDebertaPreTrainedModel, TFMaskedLanguageModelingLoss):
def __init__(self, config: DebertaConfig, *inputs, **kwargs):
super().__init__(config, *inputs, **kwargs)
@@ -1185,8 +1186,9 @@ class TFDebertaForMaskedLM(TFDebertaPreTrainedModel, TFMaskedLanguageModelingLos
) -> Union[TFMaskedLMOutput, Tuple[tf.Tensor]]:
r"""
labels (`tf.Tensor` or `np.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
"""
inputs = input_processing(
func=self.call,
@@ -1286,8 +1288,9 @@ class TFDebertaForSequenceClassification(TFDebertaPreTrainedModel, TFSequenceCla
) -> Union[TFSequenceClassifierOutput, Tuple[tf.Tensor]]:
r"""
labels (`tf.Tensor` or `np.ndarray` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
inputs = input_processing(
func=self.call,
@@ -1476,12 +1479,12 @@ class TFDebertaForQuestionAnswering(TFDebertaPreTrainedModel, TFQuestionAnswerin
r"""
start_positions (`tf.Tensor` or `np.ndarray` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the start of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
end_positions (`tf.Tensor` or `np.ndarray` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the end of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
"""
inputs = input_processing(
func=self.call,
diff --git a/src/transformers/models/deberta/tokenization_deberta.py b/src/transformers/models/deberta/tokenization_deberta.py
index 97ddff5d5e..6bca0ed581 100644
--- a/src/transformers/models/deberta/tokenization_deberta.py
+++ b/src/transformers/models/deberta/tokenization_deberta.py
@@ -203,8 +203,7 @@ class DebertaTokenizer(GPT2Tokenizer):
Optional second list of IDs for sequence pairs.
Returns:
- `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given
- sequence(s).
+ `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
"""
sep = [self.sep_token_id]
cls = [self.cls_token_id]
diff --git a/src/transformers/models/deberta/tokenization_deberta_fast.py b/src/transformers/models/deberta/tokenization_deberta_fast.py
index ae8e37abcc..74c2e4aca2 100644
--- a/src/transformers/models/deberta/tokenization_deberta_fast.py
+++ b/src/transformers/models/deberta/tokenization_deberta_fast.py
@@ -129,8 +129,8 @@ class DebertaTokenizerFast(GPT2TokenizerFast):
@property
def mask_token(self) -> str:
"""
- `str`: Mask token, to use when training a model with masked-language modeling. Log an error if used while
- not having been set.
+ `str`: Mask token, to use when training a model with masked-language modeling. Log an error if used while not
+ having been set.
Deberta tokenizer has a special mask token to be used in the fill-mask pipeline. The mask token will greedily
comprise the space before the *[MASK]*.
@@ -196,8 +196,7 @@ class DebertaTokenizerFast(GPT2TokenizerFast):
Optional second list of IDs for sequence pairs.
Returns:
- `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given
- sequence(s).
+ `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
"""
sep = [self.sep_token_id]
cls = [self.cls_token_id]
diff --git a/src/transformers/models/deberta_v2/configuration_deberta_v2.py b/src/transformers/models/deberta_v2/configuration_deberta_v2.py
index 1c283f2cfa..f7fbcd2dab 100644
--- a/src/transformers/models/deberta_v2/configuration_deberta_v2.py
+++ b/src/transformers/models/deberta_v2/configuration_deberta_v2.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" DeBERTa-v2 model configuration """
+""" DeBERTa-v2 model configuration"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging
@@ -30,13 +30,13 @@ DEBERTA_V2_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class DebertaV2Config(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`DebertaV2Model`]. It is used
- to instantiate a DeBERTa-v2 model according to the specified arguments, defining the model architecture.
- Instantiating a configuration with the defaults will yield a similar configuration to that of the DeBERTa
+ This is the configuration class to store the configuration of a [`DebertaV2Model`]. It is used to instantiate a
+ DeBERTa-v2 model according to the specified arguments, defining the model architecture. Instantiating a
+ configuration with the defaults will yield a similar configuration to that of the DeBERTa
[microsoft/deberta-v2-xlarge](https://huggingface.co/microsoft/deberta-base) architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Arguments:
vocab_size (`int`, *optional*, defaults to 128100):
@@ -51,9 +51,9 @@ class DebertaV2Config(PretrainedConfig):
intermediate_size (`int`, *optional*, defaults to 6144):
Dimensionality of the "intermediate" (often named feed-forward) layer in the Transformer encoder.
hidden_act (`str` or `Callable`, *optional*, defaults to `"gelu"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"silu"`, `"gelu"`, `"tanh"`, `"gelu_fast"`,
- `"mish"`, `"linear"`, `"sigmoid"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"silu"`, `"gelu"`, `"tanh"`, `"gelu_fast"`, `"mish"`, `"linear"`, `"sigmoid"` and `"gelu_new"`
+ are supported.
hidden_dropout_prob (`float`, *optional*, defaults to 0.1):
The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
attention_probs_dropout_prob (`float`, *optional*, defaults to 0.1):
@@ -62,8 +62,7 @@ class DebertaV2Config(PretrainedConfig):
The maximum sequence length that this model might ever be used with. Typically set this to something large
just in case (e.g., 512 or 1024 or 2048).
type_vocab_size (`int`, *optional*, defaults to 0):
- The vocabulary size of the `token_type_ids` passed when calling [`DebertaModel`] or
- [`TFDebertaModel`].
+ The vocabulary size of the `token_type_ids` passed when calling [`DebertaModel`] or [`TFDebertaModel`].
initializer_range (`float`, *optional*, defaults to 0.02):
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
layer_norm_eps (`float`, *optional*, defaults to 1e-7):
@@ -71,8 +70,8 @@ class DebertaV2Config(PretrainedConfig):
relative_attention (`bool`, *optional*, defaults to `True`):
Whether use relative position encoding.
max_relative_positions (`int`, *optional*, defaults to -1):
- The range of relative positions `[-max_position_embeddings, max_position_embeddings]`. Use the same
- value as `max_position_embeddings`.
+ The range of relative positions `[-max_position_embeddings, max_position_embeddings]`. Use the same value
+ as `max_position_embeddings`.
pad_token_id (`int`, *optional*, defaults to 0):
The value used to pad input_ids.
position_biased_input (`bool`, *optional*, defaults to `False`):
diff --git a/src/transformers/models/deberta_v2/modeling_deberta_v2.py b/src/transformers/models/deberta_v2/modeling_deberta_v2.py
index 19318c7be5..be6e93e973 100644
--- a/src/transformers/models/deberta_v2/modeling_deberta_v2.py
+++ b/src/transformers/models/deberta_v2/modeling_deberta_v2.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" PyTorch DeBERTa-v2 model. """
+""" PyTorch DeBERTa-v2 model."""
import math
from collections.abc import Sequence
@@ -80,7 +80,8 @@ class XSoftmax(torch.autograd.Function):
Args:
input (`torch.tensor`): The input tensor that will apply softmax.
- mask (`torch.IntTensor`): The mask matrix where 0 indicate that element will be ignored in the softmax calculation.
+ mask (`torch.IntTensor`):
+ The mask matrix where 0 indicate that element will be ignored in the softmax calculation.
dim (int): The dimension that will apply softmax
Example:
@@ -542,8 +543,8 @@ def build_relative_position(query_size, key_size, bucket_size=-1, max_position=-
Build relative position according to the query and key
We assume the absolute position of query \\(P_q\\) is range from (0, query_size) and the absolute position of key
- \\(P_k\\) is range from (0, key_size), The relative positions from query to key is \\(R_{q \\rightarrow k} =
- P_q - P_k\\)
+ \\(P_k\\) is range from (0, key_size), The relative positions from query to key is \\(R_{q \\rightarrow k} = P_q -
+ P_k\\)
Args:
query_size (int): the length of query
@@ -925,20 +926,20 @@ class DebertaV2PreTrainedModel(PreTrainedModel):
DEBERTA_START_DOCSTRING = r"""
- The DeBERTa model was proposed in [DeBERTa: Decoding-enhanced BERT with Disentangled Attention](https://arxiv.org/abs/2006.03654) by Pengcheng He, Xiaodong Liu, Jianfeng Gao, Weizhu Chen. It's build on top of
- BERT/RoBERTa with two improvements, i.e. disentangled attention and enhanced mask decoder. With those two
+ The DeBERTa model was proposed in [DeBERTa: Decoding-enhanced BERT with Disentangled
+ Attention](https://arxiv.org/abs/2006.03654) by Pengcheng He, Xiaodong Liu, Jianfeng Gao, Weizhu Chen. It's build
+ on top of BERT/RoBERTa with two improvements, i.e. disentangled attention and enhanced mask decoder. With those two
improvements, it out perform BERT/RoBERTa on a majority of tasks with 80GB pretraining data.
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.```
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.```
Parameters:
config ([`DebertaV2Config`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
DEBERTA_INPUTS_DOCSTRING = r"""
@@ -946,9 +947,8 @@ DEBERTA_INPUTS_DOCSTRING = r"""
input_ids (`torch.LongTensor` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`DebertaV2Tokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`DebertaV2Tokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
@@ -959,20 +959,22 @@ DEBERTA_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert *input_ids* indices into associated vectors
- than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -1100,7 +1102,7 @@ class DebertaV2Model(DebertaV2PreTrainedModel):
)
-@add_start_docstrings("""DeBERTa Model with a `language modeling` head on top. """, DEBERTA_START_DOCSTRING)
+@add_start_docstrings("""DeBERTa Model with a `language modeling` head on top.""", DEBERTA_START_DOCSTRING)
# Copied from transformers.models.deberta.modeling_deberta.DebertaForMaskedLM with Deberta->DebertaV2
class DebertaV2ForMaskedLM(DebertaV2PreTrainedModel):
_keys_to_ignore_on_load_unexpected = [r"pooler"]
@@ -1142,8 +1144,9 @@ class DebertaV2ForMaskedLM(DebertaV2PreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1283,8 +1286,9 @@ class DebertaV2ForSequenceClassification(DebertaV2PreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1478,12 +1482,12 @@ class DebertaV2ForQuestionAnswering(DebertaV2PreTrainedModel):
r"""
start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the start of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the end of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
diff --git a/src/transformers/models/deberta_v2/modeling_tf_deberta_v2.py b/src/transformers/models/deberta_v2/modeling_tf_deberta_v2.py
index f5e034df33..571cbb7297 100644
--- a/src/transformers/models/deberta_v2/modeling_tf_deberta_v2.py
+++ b/src/transformers/models/deberta_v2/modeling_tf_deberta_v2.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" TF 2.0 DeBERTa-v2 model. """
+""" TF 2.0 DeBERTa-v2 model."""
from typing import Dict, Optional, Tuple, Union
@@ -455,7 +455,6 @@ class TFDebertaV2Encoder(tf.keras.layers.Layer):
def make_log_bucket_position(relative_pos, bucket_size, max_position):
- """ """
sign = tf.math.sign(relative_pos)
mid = bucket_size // 2
abs_pos = tf.where((relative_pos < mid) & (relative_pos > -mid), mid - 1, tf.math.abs(relative_pos))
@@ -476,8 +475,8 @@ def build_relative_position(query_size, key_size, bucket_size=-1, max_position=-
Build relative position according to the query and key
We assume the absolute position of query \\(P_q\\) is range from (0, query_size) and the absolute position of key
- \\(P_k\\) is range from (0, key_size), The relative positions from query to key is \\(R_{q \\rightarrow k} =
- P_q - P_k\\)
+ \\(P_k\\) is range from (0, key_size), The relative positions from query to key is \\(R_{q \\rightarrow k} = P_q -
+ P_k\\)
Args:
query_size (int): the length of query
@@ -1126,13 +1125,14 @@ class TFDebertaV2PreTrainedModel(TFPreTrainedModel):
DEBERTA_START_DOCSTRING = r"""
- The DeBERTa model was proposed in [DeBERTa: Decoding-enhanced BERT with Disentangled Attention](https://arxiv.org/abs/2006.03654) by Pengcheng He, Xiaodong Liu, Jianfeng Gao, Weizhu Chen. It's build on top of
- BERT/RoBERTa with two improvements, i.e. disentangled attention and enhanced mask decoder. With those two
+ The DeBERTa model was proposed in [DeBERTa: Decoding-enhanced BERT with Disentangled
+ Attention](https://arxiv.org/abs/2006.03654) by Pengcheng He, Xiaodong Liu, Jianfeng Gao, Weizhu Chen. It's build
+ on top of BERT/RoBERTa with two improvements, i.e. disentangled attention and enhanced mask decoder. With those two
improvements, it out perform BERT/RoBERTa on a majority of tasks with 80GB pretraining data.
- This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use
- it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage
- and behavior.
+ This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it
+ as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and
+ behavior.
@@ -1141,11 +1141,11 @@ DEBERTA_START_DOCSTRING = r"""
- having all inputs as keyword arguments (like PyTorch models), or
- having all inputs as a list, tuple or dict in the first positional arguments.
- This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all
- the tensors in the first argument of the model call function: `model(inputs)`.
+ This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all the
+ tensors in the first argument of the model call function: `model(inputs)`.
- If you choose this second option, there are three possibilities you can use to gather all the input Tensors in
- the first positional argument :
+ If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the
+ first positional argument :
- a single Tensor with `input_ids` only and nothing else: `model(inputs_ids)`
- a list of varying length with one or several input Tensors IN THE ORDER given in the docstring:
@@ -1158,18 +1158,16 @@ DEBERTA_START_DOCSTRING = r"""
Parameters:
config ([`DebertaV2Config`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
DEBERTA_INPUTS_DOCSTRING = r"""
Args:
- input_ids (`np.ndarray`, `tf.Tensor`, `List[tf.Tensor]` ``Dict[str, tf.Tensor]` or `Dict[str, np.ndarray]` and each example must have the shape `({0})`)
+ input_ids (`np.ndarray`, `tf.Tensor`, `List[tf.Tensor]` ``Dict[str, tf.Tensor]` or `Dict[str, np.ndarray]` and each example must have the shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`DebertaV2Tokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`DebertaV2Tokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`np.ndarray` or `tf.Tensor` of shape `({0})`, *optional*):
@@ -1180,20 +1178,22 @@ DEBERTA_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`np.ndarray` or `tf.Tensor` of shape `({0})`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`np.ndarray` or `tf.Tensor` of shape `({0})`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
inputs_embeds (`np.ndarray` or `tf.Tensor` of shape `({0}, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert *input_ids* indices into associated vectors
- than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -1271,7 +1271,7 @@ class TFDebertaV2Model(TFDebertaV2PreTrainedModel):
return TFBaseModelOutput(last_hidden_state=output.last_hidden_state, hidden_states=hs, attentions=attns)
-@add_start_docstrings("""DeBERTa Model with a `language modeling` head on top. """, DEBERTA_START_DOCSTRING)
+@add_start_docstrings("""DeBERTa Model with a `language modeling` head on top.""", DEBERTA_START_DOCSTRING)
# Copied from transformers.models.deberta.modeling_tf_deberta.TFDebertaForMaskedLM with Deberta->DebertaV2
class TFDebertaV2ForMaskedLM(TFDebertaV2PreTrainedModel, TFMaskedLanguageModelingLoss):
def __init__(self, config: DebertaV2Config, *inputs, **kwargs):
@@ -1312,8 +1312,9 @@ class TFDebertaV2ForMaskedLM(TFDebertaV2PreTrainedModel, TFMaskedLanguageModelin
) -> Union[TFMaskedLMOutput, Tuple[tf.Tensor]]:
r"""
labels (`tf.Tensor` or `np.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
"""
inputs = input_processing(
func=self.call,
@@ -1414,8 +1415,9 @@ class TFDebertaV2ForSequenceClassification(TFDebertaV2PreTrainedModel, TFSequenc
) -> Union[TFSequenceClassifierOutput, Tuple[tf.Tensor]]:
r"""
labels (`tf.Tensor` or `np.ndarray` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
inputs = input_processing(
func=self.call,
@@ -1606,12 +1608,12 @@ class TFDebertaV2ForQuestionAnswering(TFDebertaV2PreTrainedModel, TFQuestionAnsw
r"""
start_positions (`tf.Tensor` or `np.ndarray` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the start of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
end_positions (`tf.Tensor` or `np.ndarray` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the end of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
"""
inputs = input_processing(
func=self.call,
diff --git a/src/transformers/models/deberta_v2/tokenization_deberta_v2.py b/src/transformers/models/deberta_v2/tokenization_deberta_v2.py
index 5c6612af1a..c28ba10195 100644
--- a/src/transformers/models/deberta_v2/tokenization_deberta_v2.py
+++ b/src/transformers/models/deberta_v2/tokenization_deberta_v2.py
@@ -83,7 +83,9 @@ class DebertaV2Tokenizer(PreTrainedTokenizer):
The token used for masking values. This is the token used when training this model with masked language
modeling. This is the token which the model will try to predict.
sp_model_kwargs (`dict`, *optional*):
- Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things, to set:
+ Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for
+ SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things,
+ to set:
- `enable_sampling`: Enable subword regularization.
- `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout.
@@ -242,8 +244,7 @@ class DebertaV2Tokenizer(PreTrainedTokenizer):
Optional second list of IDs for sequence pairs.
Returns:
- `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given
- sequence(s).
+ `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
"""
sep = [self.sep_token_id]
cls = [self.cls_token_id]
@@ -270,7 +271,9 @@ class SPMTokenizer:
[SentencePiece](https://github.com/google/sentencepiece) file (generally has a *.spm* extension) that
contains the vocabulary necessary to instantiate a tokenizer.
sp_model_kwargs (`dict`, *optional*):
- Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things, to set:
+ Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for
+ SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things,
+ to set:
- `enable_sampling`: Enable subword regularization.
- `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout.
diff --git a/src/transformers/models/deit/configuration_deit.py b/src/transformers/models/deit/configuration_deit.py
index 09d979daef..4e5968cf1e 100644
--- a/src/transformers/models/deit/configuration_deit.py
+++ b/src/transformers/models/deit/configuration_deit.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" DeiT model configuration """
+""" DeiT model configuration"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging
@@ -28,14 +28,14 @@ DEIT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class DeiTConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`DeiTModel`]. It is used to
- instantiate an DeiT model according to the specified arguments, defining the model architecture. Instantiating a
- configuration with the defaults will yield a similar configuration to that of the DeiT
+ This is the configuration class to store the configuration of a [`DeiTModel`]. It is used to instantiate an DeiT
+ model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
+ defaults will yield a similar configuration to that of the DeiT
[facebook/deit-base-distilled-patch16-224](https://huggingface.co/facebook/deit-base-distilled-patch16-224)
architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
@@ -48,8 +48,8 @@ class DeiTConfig(PretrainedConfig):
intermediate_size (`int`, *optional*, defaults to 3072):
Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"selu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"selu"` and `"gelu_new"` are supported.
hidden_dropout_prob (`float`, *optional*, defaults to 0.1):
The dropout probabilitiy for all fully connected layers in the embeddings, encoder, and pooler.
attention_probs_dropout_prob (`float`, *optional*, defaults to 0.1):
diff --git a/src/transformers/models/deit/feature_extraction_deit.py b/src/transformers/models/deit/feature_extraction_deit.py
index a5ed140303..acb83a3c57 100644
--- a/src/transformers/models/deit/feature_extraction_deit.py
+++ b/src/transformers/models/deit/feature_extraction_deit.py
@@ -38,26 +38,25 @@ class DeiTFeatureExtractor(FeatureExtractionMixin, ImageFeatureExtractionMixin):
r"""
Constructs a DeiT feature extractor.
- This feature extractor inherits from [`FeatureExtractionMixin`] which contains most of the main
- methods. Users should refer to this superclass for more information regarding those methods.
+ This feature extractor inherits from [`FeatureExtractionMixin`] which contains most of the main methods. Users
+ should refer to this superclass for more information regarding those methods.
Args:
do_resize (`bool`, *optional*, defaults to `True`):
Whether to resize the input to a certain `size`.
size (`int` or `Tuple(int)`, *optional*, defaults to 256):
Resize the input to the given size. If a tuple is provided, it should be (width, height). If only an
- integer is provided, then the input will be resized to (size, size). Only has an effect if `do_resize`
- is set to `True`.
+ integer is provided, then the input will be resized to (size, size). Only has an effect if `do_resize` is
+ set to `True`.
resample (`int`, *optional*, defaults to `PIL.Image.BICUBIC`):
An optional resampling filter. This can be one of `PIL.Image.NEAREST`, `PIL.Image.BOX`,
- `PIL.Image.BILINEAR`, `PIL.Image.HAMMING`, `PIL.Image.BICUBIC` or `PIL.Image.LANCZOS`.
- Only has an effect if `do_resize` is set to `True`.
+ `PIL.Image.BILINEAR`, `PIL.Image.HAMMING`, `PIL.Image.BICUBIC` or `PIL.Image.LANCZOS`. Only has an effect
+ if `do_resize` is set to `True`.
do_center_crop (`bool`, *optional*, defaults to `True`):
- Whether to crop the input at the center. If the input size is smaller than `crop_size` along any edge,
- the image is padded with 0's and then center cropped.
+ Whether to crop the input at the center. If the input size is smaller than `crop_size` along any edge, the
+ image is padded with 0's and then center cropped.
crop_size (`int`, *optional*, defaults to 224):
- Desired output size when applying center-cropping. Only has an effect if `do_center_crop` is set to
- `True`.
+ Desired output size when applying center-cropping. Only has an effect if `do_center_crop` is set to `True`.
do_normalize (`bool`, *optional*, defaults to `True`):
Whether or not to normalize the input with `image_mean` and `image_std`.
image_mean (`List[int]`, defaults to `[0.485, 0.456, 0.406]`):
diff --git a/src/transformers/models/deit/modeling_deit.py b/src/transformers/models/deit/modeling_deit.py
index 9ca4bd479b..acad311b34 100644
--- a/src/transformers/models/deit/modeling_deit.py
+++ b/src/transformers/models/deit/modeling_deit.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" PyTorch DeiT model. """
+""" PyTorch DeiT model."""
import collections.abc
@@ -410,15 +410,14 @@ class DeiTPreTrainedModel(PreTrainedModel):
DEIT_START_DOCSTRING = r"""
- This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use
- it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
+ This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it
+ as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
behavior.
Parameters:
config ([`DeiTConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
DEIT_INPUTS_DOCSTRING = r"""
@@ -592,8 +591,9 @@ class DeiTForImageClassification(DeiTPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the image classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
Returns:
@@ -671,12 +671,13 @@ class DeiTForImageClassificationWithTeacherOutput(ModelOutput):
Prediction scores of the distillation head (i.e. the linear layer on top of the final hidden state of the
distillation token).
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of
- each layer plus the initial embedding outputs.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer
+ plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the
- weighted average in the self-attention heads.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
+ the self-attention heads.
"""
logits: torch.FloatTensor = None
diff --git a/src/transformers/models/detr/configuration_detr.py b/src/transformers/models/detr/configuration_detr.py
index 3edc9da80e..a93ac62e74 100644
--- a/src/transformers/models/detr/configuration_detr.py
+++ b/src/transformers/models/detr/configuration_detr.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" DETR model configuration """
+""" DETR model configuration"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging
@@ -28,18 +28,19 @@ DETR_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class DetrConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`DetrModel`]. It is used to
- instantiate a DETR model according to the specified arguments, defining the model architecture. Instantiating a
- configuration with the defaults will yield a similar configuration to that of the DETR [facebook/detr-resnet-50](https://huggingface.co/facebook/detr-resnet-50) architecture.
+ This is the configuration class to store the configuration of a [`DetrModel`]. It is used to instantiate a DETR
+ model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
+ defaults will yield a similar configuration to that of the DETR
+ [facebook/detr-resnet-50](https://huggingface.co/facebook/detr-resnet-50) architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
num_queries (`int`, *optional*, defaults to 100):
- Number of object queries, i.e. detection slots. This is the maximal number of objects
- [`DetrModel`] can detect in a single image. For COCO, we recommend 100 queries.
+ Number of object queries, i.e. detection slots. This is the maximal number of objects [`DetrModel`] can
+ detect in a single image. For COCO, we recommend 100 queries.
d_model (`int`, *optional*, defaults to 256):
Dimension of the layers.
encoder_layers (`int`, *optional*, defaults to 6):
@@ -55,8 +56,8 @@ class DetrConfig(PretrainedConfig):
encoder_ffn_dim (`int`, *optional*, defaults to 2048):
Dimension of the "intermediate" (often named feed-forward) layer in decoder.
activation_function (`str` or `function`, *optional*, defaults to `"relu"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"silu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"silu"` and `"gelu_new"` are supported.
dropout (`float`, *optional*, defaults to 0.1):
The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
attention_dropout (`float`, *optional*, defaults to 0.0):
@@ -68,19 +69,19 @@ class DetrConfig(PretrainedConfig):
init_xavier_std (`float`, *optional*, defaults to 1):
The scaling factor used for the Xavier initialization gain in the HM Attention map module.
encoder_layerdrop: (`float`, *optional*, defaults to 0.0):
- The LayerDrop probability for the encoder. See the [LayerDrop paper](see
- https://arxiv.org/abs/1909.11556) for more details.
+ The LayerDrop probability for the encoder. See the [LayerDrop paper](see https://arxiv.org/abs/1909.11556)
+ for more details.
decoder_layerdrop: (`float`, *optional*, defaults to 0.0):
- The LayerDrop probability for the decoder. See the [LayerDrop paper](see
- https://arxiv.org/abs/1909.11556) for more details.
+ The LayerDrop probability for the decoder. See the [LayerDrop paper](see https://arxiv.org/abs/1909.11556)
+ for more details.
auxiliary_loss (`bool`, *optional*, defaults to `False`):
Whether auxiliary decoding losses (loss at each decoder layer) are to be used.
position_embedding_type (`str`, *optional*, defaults to `"sine"`):
- Type of position embeddings to be used on top of the image features. One of `"sine"` or
- `"learned"`.
+ Type of position embeddings to be used on top of the image features. One of `"sine"` or `"learned"`.
backbone (`str`, *optional*, defaults to `"resnet50"`):
Name of convolutional backbone to use. Supports any convolutional backbone from the timm package. For a
- list of all available models, see [this page](https://rwightman.github.io/pytorch-image-models/#load-a-pretrained-model).
+ list of all available models, see [this
+ page](https://rwightman.github.io/pytorch-image-models/#load-a-pretrained-model).
dilation (`bool`, *optional*, defaults to `False`):
Whether to replace stride with dilation in the last convolutional block (DC5).
class_cost (`float`, *optional*, defaults to 1):
diff --git a/src/transformers/models/detr/feature_extraction_detr.py b/src/transformers/models/detr/feature_extraction_detr.py
index a2f93ac2a2..fef0fc4a43 100644
--- a/src/transformers/models/detr/feature_extraction_detr.py
+++ b/src/transformers/models/detr/feature_extraction_detr.py
@@ -124,8 +124,8 @@ class DetrFeatureExtractor(FeatureExtractionMixin, ImageFeatureExtractionMixin):
r"""
Constructs a DETR feature extractor.
- This feature extractor inherits from [`FeatureExtractionMixin`] which contains most of the main
- methods. Users should refer to this superclass for more information regarding those methods.
+ This feature extractor inherits from [`FeatureExtractionMixin`] which contains most of the main methods. Users
+ should refer to this superclass for more information regarding those methods.
Args:
@@ -134,13 +134,13 @@ class DetrFeatureExtractor(FeatureExtractionMixin, ImageFeatureExtractionMixin):
do_resize (`bool`, *optional*, defaults to `True`):
Whether to resize the input to a certain `size`.
size (`int`, *optional*, defaults to 800):
- Resize the input to the given size. Only has an effect if `do_resize` is set to `True`. If size
- is a sequence like `(width, height)`, output size will be matched to this. If size is an int, smaller
- edge of the image will be matched to this number. i.e, if `height > width`, then image will be
- rescaled to `(size * height / width, size)`.
+ Resize the input to the given size. Only has an effect if `do_resize` is set to `True`. If size is a
+ sequence like `(width, height)`, output size will be matched to this. If size is an int, smaller edge of
+ the image will be matched to this number. i.e, if `height > width`, then image will be rescaled to `(size *
+ height / width, size)`.
max_size (`int`, *optional*, defaults to `1333`):
- The largest size an image dimension can have (otherwise it's capped). Only has an effect if
- `do_resize` is set to `True`.
+ The largest size an image dimension can have (otherwise it's capped). Only has an effect if `do_resize` is
+ set to `True`.
do_normalize (`bool`, *optional*, defaults to `True`):
Whether or not to normalize the input with mean and standard deviation.
image_mean (`int`, *optional*, defaults to `[0.485, 0.456, 0.406]`):
@@ -432,15 +432,17 @@ class DetrFeatureExtractor(FeatureExtractionMixin, ImageFeatureExtractionMixin):
annotations (`Dict`, `List[Dict]`, *optional*):
The corresponding annotations in COCO format.
- In case [`DetrFeatureExtractor`] was initialized with `format = "coco_detection"`, the annotations for each image should have the following format: {'image_id': int,
- 'annotations': [annotation]}, with the annotations being a list of COCO object annotations.
+ In case [`DetrFeatureExtractor`] was initialized with `format = "coco_detection"`, the annotations for
+ each image should have the following format: {'image_id': int, 'annotations': [annotation]}, with the
+ annotations being a list of COCO object annotations.
- In case [`DetrFeatureExtractor`] was initialized with `format = "coco_panoptic"`, the annotations for each image should have the following format: {'image_id': int,
- 'file_name': str, 'segments_info': [segment_info]} with segments_info being a list of COCO panoptic
- annotations.
+ In case [`DetrFeatureExtractor`] was initialized with `format = "coco_panoptic"`, the annotations for
+ each image should have the following format: {'image_id': int, 'file_name': str, 'segments_info':
+ [segment_info]} with segments_info being a list of COCO panoptic annotations.
return_segmentation_masks (`Dict`, `List[Dict]`, *optional*, defaults to `False`):
- Whether to also include instance segmentation masks as part of the labels in case `format = "coco_detection"`.
+ Whether to also include instance segmentation masks as part of the labels in case `format =
+ "coco_detection"`.
masks_path (`pathlib.Path`, *optional*):
Path to the directory containing the PNG files that store the class-agnostic image segmentations. Only
@@ -455,8 +457,8 @@ class DetrFeatureExtractor(FeatureExtractionMixin, ImageFeatureExtractionMixin):
- 0 for pixels that are padding (i.e. **masked**).
return_tensors (`str` or [`~file_utils.TensorType`], *optional*):
- If set, will return tensors instead of NumPy arrays. If set to `'pt'`, return PyTorch
- `torch.Tensor` objects.
+ If set, will return tensors instead of NumPy arrays. If set to `'pt'`, return PyTorch `torch.Tensor`
+ objects.
Returns:
[`BatchFeature`]: A [`BatchFeature`] with the following fields:
@@ -638,8 +640,8 @@ class DetrFeatureExtractor(FeatureExtractionMixin, ImageFeatureExtractionMixin):
pixel_values_list (`List[torch.Tensor]`):
List of images (pixel values) to be padded. Each image should be a tensor of shape (C, H, W).
return_tensors (`str` or [`~file_utils.TensorType`], *optional*):
- If set, will return tensors instead of NumPy arrays. If set to `'pt'`, return PyTorch
- `torch.Tensor` objects.
+ If set, will return tensors instead of NumPy arrays. If set to `'pt'`, return PyTorch `torch.Tensor`
+ objects.
Returns:
[`BatchFeature`]: A [`BatchFeature`] with the following fields:
@@ -674,8 +676,8 @@ class DetrFeatureExtractor(FeatureExtractionMixin, ImageFeatureExtractionMixin):
# inspired by https://github.com/facebookresearch/detr/blob/master/models/detr.py#L258
def post_process(self, outputs, target_sizes):
"""
- Converts the output of [`DetrForObjectDetection`] into the format expected by the COCO api.
- Only supports PyTorch.
+ Converts the output of [`DetrForObjectDetection`] into the format expected by the COCO api. Only supports
+ PyTorch.
Args:
outputs ([`DetrObjectDetectionOutput`]):
@@ -686,8 +688,8 @@ class DetrFeatureExtractor(FeatureExtractionMixin, ImageFeatureExtractionMixin):
augment, but before padding.
Returns:
- `List[Dict]`: A list of dictionaries, each dictionary containing the scores, labels and boxes for an
- image in the batch as predicted by the model.
+ `List[Dict]`: A list of dictionaries, each dictionary containing the scores, labels and boxes for an image
+ in the batch as predicted by the model.
"""
out_logits, out_bbox = outputs.logits, outputs.pred_boxes
@@ -712,8 +714,7 @@ class DetrFeatureExtractor(FeatureExtractionMixin, ImageFeatureExtractionMixin):
def post_process_segmentation(self, outputs, target_sizes, threshold=0.9, mask_threshold=0.5):
"""
- Converts the output of [`DetrForSegmentation`] into image segmentation predictions. Only
- supports PyTorch.
+ Converts the output of [`DetrForSegmentation`] into image segmentation predictions. Only supports PyTorch.
Parameters:
outputs ([`DetrSegmentationOutput`]):
@@ -726,8 +727,8 @@ class DetrFeatureExtractor(FeatureExtractionMixin, ImageFeatureExtractionMixin):
Threshold to use when turning the predicted masks into binary values.
Returns:
- `List[Dict]`: A list of dictionaries, each dictionary containing the scores, labels, and masks for an
- image in the batch as predicted by the model.
+ `List[Dict]`: A list of dictionaries, each dictionary containing the scores, labels, and masks for an image
+ in the batch as predicted by the model.
"""
out_logits, raw_masks = outputs.logits, outputs.pred_masks
preds = []
@@ -755,13 +756,13 @@ class DetrFeatureExtractor(FeatureExtractionMixin, ImageFeatureExtractionMixin):
# inspired by https://github.com/facebookresearch/detr/blob/master/models/segmentation.py#L218
def post_process_instance(self, results, outputs, orig_target_sizes, max_target_sizes, threshold=0.5):
"""
- Converts the output of [`DetrForSegmentation`] into actual instance segmentation
- predictions. Only supports PyTorch.
+ Converts the output of [`DetrForSegmentation`] into actual instance segmentation predictions. Only supports
+ PyTorch.
Args:
results (`List[Dict]`):
- Results list obtained by [`~DetrFeatureExtractor.post_process`], to which "masks"
- results will be added.
+ Results list obtained by [`~DetrFeatureExtractor.post_process`], to which "masks" results will be
+ added.
outputs ([`DetrSegmentationOutput`]):
Raw outputs of the model.
orig_target_sizes (`torch.Tensor` of shape `(batch_size, 2)`):
@@ -774,8 +775,8 @@ class DetrFeatureExtractor(FeatureExtractionMixin, ImageFeatureExtractionMixin):
Threshold to use when turning the predicted masks into binary values.
Returns:
- `List[Dict]`: A list of dictionaries, each dictionary containing the scores, labels, boxes and masks
- for an image in the batch as predicted by the model.
+ `List[Dict]`: A list of dictionaries, each dictionary containing the scores, labels, boxes and masks for an
+ image in the batch as predicted by the model.
"""
if len(orig_target_sizes) != len(max_target_sizes):
@@ -799,8 +800,7 @@ class DetrFeatureExtractor(FeatureExtractionMixin, ImageFeatureExtractionMixin):
# inspired by https://github.com/facebookresearch/detr/blob/master/models/segmentation.py#L241
def post_process_panoptic(self, outputs, processed_sizes, target_sizes=None, is_thing_map=None, threshold=0.85):
"""
- Converts the output of [`DetrForSegmentation`] into actual panoptic predictions. Only
- supports PyTorch.
+ Converts the output of [`DetrForSegmentation`] into actual panoptic predictions. Only supports PyTorch.
Parameters:
outputs ([`DetrSegmentationOutput`]):
@@ -818,8 +818,8 @@ class DetrFeatureExtractor(FeatureExtractionMixin, ImageFeatureExtractionMixin):
Threshold to use to filter out queries.
Returns:
- `List[Dict]`: A list of dictionaries, each dictionary containing a PNG string and segments_info values
- for an image in the batch as predicted by the model.
+ `List[Dict]`: A list of dictionaries, each dictionary containing a PNG string and segments_info values for
+ an image in the batch as predicted by the model.
"""
if target_sizes is None:
target_sizes = processed_sizes
diff --git a/src/transformers/models/detr/modeling_detr.py b/src/transformers/models/detr/modeling_detr.py
index 7d1140577a..b05fd1d56c 100644
--- a/src/transformers/models/detr/modeling_detr.py
+++ b/src/transformers/models/detr/modeling_detr.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" PyTorch DETR model. """
+""" PyTorch DETR model."""
import math
@@ -71,15 +71,17 @@ class DetrDecoderOutput(BaseModelOutputWithCrossAttentions):
last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
Sequence of hidden-states at the output of the last layer of the model.
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of
- each layer plus the initial embedding outputs.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer
+ plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the
- weighted average in the self-attention heads.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
+ the self-attention heads.
cross_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` and `config.add_cross_attention=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights of the decoder's cross-attention layer, after the
- attention softmax, used to compute the weighted average in the cross-attention heads.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`. Attentions weights of the decoder's cross-attention layer, after the attention softmax,
+ used to compute the weighted average in the cross-attention heads.
intermediate_hidden_states (`torch.FloatTensor` of shape `(config.decoder_layers, batch_size, num_queries, hidden_size)`, *optional*, returned when `config.auxiliary_loss=True`):
Intermediate decoder activations, i.e. the output of each decoder layer, each of them gone through a
layernorm.
@@ -99,24 +101,27 @@ class DetrModelOutput(Seq2SeqModelOutput):
last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
Sequence of hidden-states at the output of the last layer of the decoder of the model.
decoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the decoder at the output of
- each layer plus the initial embedding outputs.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the decoder at the output of each
+ layer plus the initial embedding outputs.
decoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights of the decoder, after the attention softmax, used to
- compute the weighted average in the self-attention heads.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`. Attentions weights of the decoder, after the attention softmax, used to compute the
+ weighted average in the self-attention heads.
cross_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights of the decoder's cross-attention layer, after the
- attention softmax, used to compute the weighted average in the cross-attention heads.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`. Attentions weights of the decoder's cross-attention layer, after the attention softmax,
+ used to compute the weighted average in the cross-attention heads.
encoder_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
Sequence of hidden-states at the output of the last layer of the encoder of the model.
encoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the encoder at the output of
- each layer plus the initial embedding outputs.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the encoder at the output of each
+ layer plus the initial embedding outputs.
encoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights of the encoder, after the attention softmax, used to
- compute the weighted average in the self-attention heads.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`. Attentions weights of the encoder, after the attention softmax, used to compute the
+ weighted average in the self-attention heads.
intermediate_hidden_states (`torch.FloatTensor` of shape `(config.decoder_layers, batch_size, sequence_length, hidden_size)`, *optional*, returned when `config.auxiliary_loss=True`):
Intermediate decoder activations, i.e. the output of each decoder layer, each of them gone through a
layernorm.
@@ -142,33 +147,36 @@ class DetrObjectDetectionOutput(ModelOutput):
pred_boxes (`torch.FloatTensor` of shape `(batch_size, num_queries, 4)`):
Normalized boxes coordinates for all queries, represented as (center_x, center_y, width, height). These
values are normalized in [0, 1], relative to the size of each individual image in the batch (disregarding
- possible padding). You can use [`~DetrFeatureExtractor.post_process`] to retrieve the
- unnormalized bounding boxes.
+ possible padding). You can use [`~DetrFeatureExtractor.post_process`] to retrieve the unnormalized bounding
+ boxes.
auxiliary_outputs (`list[Dict]`, *optional*):
- Optional, only returned when auxilary losses are activated (i.e. `config.auxiliary_loss` is set to
- *True*) and labels are provided. It is a list of dictionaries containing the two above keys (`logits`
- and `pred_boxes`) for each decoder layer.
+ Optional, only returned when auxilary losses are activated (i.e. `config.auxiliary_loss` is set to *True*)
+ and labels are provided. It is a list of dictionaries containing the two above keys (`logits` and
+ `pred_boxes`) for each decoder layer.
last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
Sequence of hidden-states at the output of the last layer of the decoder of the model.
decoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the decoder at the output of
- each layer plus the initial embedding outputs.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the decoder at the output of each
+ layer plus the initial embedding outputs.
decoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights of the decoder, after the attention softmax, used to
- compute the weighted average in the self-attention heads.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`. Attentions weights of the decoder, after the attention softmax, used to compute the
+ weighted average in the self-attention heads.
cross_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights of the decoder's cross-attention layer, after the
- attention softmax, used to compute the weighted average in the cross-attention heads.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`. Attentions weights of the decoder's cross-attention layer, after the attention softmax,
+ used to compute the weighted average in the cross-attention heads.
encoder_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
Sequence of hidden-states at the output of the last layer of the encoder of the model.
encoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the encoder at the output of
- each layer plus the initial embedding outputs.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the encoder at the output of each
+ layer plus the initial embedding outputs.
encoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights of the encoder, after the attention softmax, used to
- compute the weighted average in the self-attention heads.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`. Attentions weights of the encoder, after the attention softmax, used to compute the
+ weighted average in the self-attention heads.
"""
loss: Optional[torch.FloatTensor] = None
@@ -202,38 +210,40 @@ class DetrSegmentationOutput(ModelOutput):
pred_boxes (`torch.FloatTensor` of shape `(batch_size, num_queries, 4)`):
Normalized boxes coordinates for all queries, represented as (center_x, center_y, width, height). These
values are normalized in [0, 1], relative to the size of each individual image in the batch (disregarding
- possible padding). You can use [`~DetrFeatureExtractor.post_process`] to retrieve the
- unnormalized bounding boxes.
+ possible padding). You can use [`~DetrFeatureExtractor.post_process`] to retrieve the unnormalized bounding
+ boxes.
pred_masks (`torch.FloatTensor` of shape `(batch_size, num_queries, height/4, width/4)`):
- Segmentation masks logits for all queries. See also
- [`~DetrFeatureExtractor.post_process_segmentation`] or
- [`~DetrFeatureExtractor.post_process_panoptic`] to evaluate instance and panoptic
- segmentation masks respectively.
+ Segmentation masks logits for all queries. See also [`~DetrFeatureExtractor.post_process_segmentation`] or
+ [`~DetrFeatureExtractor.post_process_panoptic`] to evaluate instance and panoptic segmentation masks
+ respectively.
auxiliary_outputs (`list[Dict]`, *optional*):
- Optional, only returned when auxiliary losses are activated (i.e. `config.auxiliary_loss` is set to
- *True*) and labels are provided. It is a list of dictionaries containing the two above keys (`logits`
- and `pred_boxes`) for each decoder layer.
+ Optional, only returned when auxiliary losses are activated (i.e. `config.auxiliary_loss` is set to *True*)
+ and labels are provided. It is a list of dictionaries containing the two above keys (`logits` and
+ `pred_boxes`) for each decoder layer.
last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
Sequence of hidden-states at the output of the last layer of the decoder of the model.
decoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the decoder at the output of
- each layer plus the initial embedding outputs.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the decoder at the output of each
+ layer plus the initial embedding outputs.
decoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights of the decoder, after the attention softmax, used to
- compute the weighted average in the self-attention heads.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`. Attentions weights of the decoder, after the attention softmax, used to compute the
+ weighted average in the self-attention heads.
cross_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights of the decoder's cross-attention layer, after the
- attention softmax, used to compute the weighted average in the cross-attention heads.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`. Attentions weights of the decoder's cross-attention layer, after the attention softmax,
+ used to compute the weighted average in the cross-attention heads.
encoder_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
Sequence of hidden-states at the output of the last layer of the encoder of the model.
encoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the encoder at the output of
- each layer plus the initial embedding outputs.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the encoder at the output of each
+ layer plus the initial embedding outputs.
encoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights of the encoder, after the attention softmax, used to
- compute the weighted average in the self-attention heads.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`. Attentions weights of the encoder, after the attention softmax, used to compute the
+ weighted average in the self-attention heads.
"""
loss: Optional[torch.FloatTensor] = None
@@ -692,11 +702,14 @@ class DetrDecoderLayer(nn.Module):
hidden_states (`torch.FloatTensor`): input to the layer of shape `(seq_len, batch, embed_dim)`
attention_mask (`torch.FloatTensor`): attention mask of size
`(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
- position_embeddings (`torch.FloatTensor`, *optional*): position embeddings that are added to the queries and keys
+ position_embeddings (`torch.FloatTensor`, *optional*):
+ position embeddings that are added to the queries and keys
in the cross-attention layer.
- query_position_embeddings (`torch.FloatTensor`, *optional*): position embeddings that are added to the queries and keys
+ query_position_embeddings (`torch.FloatTensor`, *optional*):
+ position embeddings that are added to the queries and keys
in the self-attention layer.
- encoder_hidden_states (`torch.FloatTensor`): cross attention input to the layer of shape `(seq_len, batch, embed_dim)`
+ encoder_hidden_states (`torch.FloatTensor`):
+ cross attention input to the layer of shape `(seq_len, batch, embed_dim)`
encoder_attention_mask (`torch.FloatTensor`): encoder attention mask of size
`(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
output_attentions (`bool`, *optional*):
@@ -804,13 +817,13 @@ class DetrPreTrainedModel(PreTrainedModel):
DETR_START_DOCSTRING = r"""
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
Parameters:
config ([`DetrConfig`]):
@@ -824,8 +837,8 @@ DETR_INPUTS_DOCSTRING = r"""
pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
Pixel values. Padding will be ignored by default should you provide it.
- Pixel values can be obtained using [`DetrFeatureExtractor`]. See
- [`DetrFeatureExtractor.__call__`] for details.
+ Pixel values can be obtained using [`DetrFeatureExtractor`]. See [`DetrFeatureExtractor.__call__`] for
+ details.
pixel_mask (`torch.LongTensor` of shape `(batch_size, height, width)`, *optional*):
Mask to avoid performing attention on padding pixel values. Mask values selected in `[0, 1]`:
@@ -838,10 +851,9 @@ DETR_INPUTS_DOCSTRING = r"""
decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, num_queries)`, *optional*):
Not used by default. Can be used to mask object queries.
encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*):
- Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*:
- `attentions`) `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`,
- *optional*) is a sequence of hidden-states at the output of the last layer of the encoder. Used in the
- cross-attention of the decoder.
+ Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`)
+ `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of
+ hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder.
inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
Optionally, instead of passing the flattened feature map (output of the backbone + projection layer), you
can choose to directly pass a flattened representation of an image.
@@ -1032,8 +1044,8 @@ class DetrDecoder(DetrPreTrainedModel):
position_embeddings (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
Position embeddings that are added to the queries and keys in each cross-attention layer.
- query_position_embeddings (`torch.FloatTensor` of shape `(batch_size, num_queries, hidden_size)`):, *optional*):
- Position embeddings that are added to the queries and keys in each self-attention layer.
+ query_position_embeddings (`torch.FloatTensor` of shape `(batch_size, num_queries, hidden_size)`):
+ , *optional*): Position embeddings that are added to the queries and keys in each self-attention layer.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under
returned tensors for more detail.
@@ -1357,7 +1369,8 @@ class DetrForObjectDetection(DetrPreTrainedModel):
labels (`List[Dict]` of len `(batch_size,)`, *optional*):
Labels for computing the bipartite matching loss. List of dicts, each dictionary containing at least the
following 2 keys: 'class_labels' and 'boxes' (the class labels and bounding boxes of an image in the batch
- respectively). The class labels themselves should be a `torch.LongTensor` of len `(number of bounding boxes in the image,)` and the boxes a `torch.FloatTensor` of shape `(number of bounding boxes in the image, 4)`.
+ respectively). The class labels themselves should be a `torch.LongTensor` of len `(number of bounding boxes
+ in the image,)` and the boxes a `torch.FloatTensor` of shape `(number of bounding boxes in the image, 4)`.
Returns:
@@ -2072,7 +2085,8 @@ class DetrHungarianMatcher(nn.Module):
Params:
class_cost: This is the relative weight of the classification error in the matching cost
- bbox_cost: This is the relative weight of the L1 error of the bounding box coordinates in the matching cost
+ bbox_cost:
+ This is the relative weight of the L1 error of the bounding box coordinates in the matching cost
giou_cost: This is the relative weight of the giou loss of the bounding box in the matching cost
"""
super().__init__()
diff --git a/src/transformers/models/distilbert/configuration_distilbert.py b/src/transformers/models/distilbert/configuration_distilbert.py
index 36e47ddea3..05027e3178 100644
--- a/src/transformers/models/distilbert/configuration_distilbert.py
+++ b/src/transformers/models/distilbert/configuration_distilbert.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" DistilBERT model configuration """
+""" DistilBERT model configuration"""
from collections import OrderedDict
from typing import Mapping
@@ -36,19 +36,18 @@ DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class DistilBertConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`DistilBertModel`] or a
- [`TFDistilBertModel`]. It is used to instantiate a DistilBERT model according to the specified
- arguments, defining the model architecture. Instantiating a configuration with the defaults will yield a similar
- configuration to that of the DistilBERT [distilbert-base-uncased](https://huggingface.co/distilbert-base-uncased) architecture.
+ This is the configuration class to store the configuration of a [`DistilBertModel`] or a [`TFDistilBertModel`]. It
+ is used to instantiate a DistilBERT model according to the specified arguments, defining the model architecture.
+ Instantiating a configuration with the defaults will yield a similar configuration to that of the DistilBERT
+ [distilbert-base-uncased](https://huggingface.co/distilbert-base-uncased) architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
vocab_size (`int`, *optional*, defaults to 30522):
Vocabulary size of the DistilBERT model. Defines the number of different tokens that can be represented by
- the `inputs_ids` passed when calling [`DistilBertModel`] or
- [`TFDistilBertModel`].
+ the `inputs_ids` passed when calling [`DistilBertModel`] or [`TFDistilBertModel`].
max_position_embeddings (`int`, *optional*, defaults to 512):
The maximum sequence length that this model might ever be used with. Typically set this to something large
just in case (e.g., 512 or 1024 or 2048).
@@ -67,13 +66,12 @@ class DistilBertConfig(PretrainedConfig):
attention_dropout (`float`, *optional*, defaults to 0.1):
The dropout ratio for the attention probabilities.
activation (`str` or `Callable`, *optional*, defaults to `"gelu"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"silu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"silu"` and `"gelu_new"` are supported.
initializer_range (`float`, *optional*, defaults to 0.02):
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
qa_dropout (`float`, *optional*, defaults to 0.1):
- The dropout probabilities used in the question answering model
- [`DistilBertForQuestionAnswering`].
+ The dropout probabilities used in the question answering model [`DistilBertForQuestionAnswering`].
seq_classif_dropout (`float`, *optional*, defaults to 0.2):
The dropout probabilities used in the sequence classification and the multiple choice model
[`DistilBertForSequenceClassification`].
diff --git a/src/transformers/models/distilbert/modeling_distilbert.py b/src/transformers/models/distilbert/modeling_distilbert.py
index db57dd061b..58f4d70ebc 100755
--- a/src/transformers/models/distilbert/modeling_distilbert.py
+++ b/src/transformers/models/distilbert/modeling_distilbert.py
@@ -377,19 +377,18 @@ class DistilBertPreTrainedModel(PreTrainedModel):
DISTILBERT_START_DOCSTRING = r"""
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
Parameters:
config ([`DistilBertConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
DISTILBERT_INPUTS_DOCSTRING = r"""
@@ -397,9 +396,8 @@ DISTILBERT_INPUTS_DOCSTRING = r"""
input_ids (`torch.LongTensor` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`DistilBertTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`DistilBertTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
@@ -416,9 +414,9 @@ DISTILBERT_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -559,7 +557,7 @@ class DistilBertModel(DistilBertPreTrainedModel):
@add_start_docstrings(
- """DistilBert Model with a `masked language modeling` head on top. """,
+ """DistilBert Model with a `masked language modeling` head on top.""",
DISTILBERT_START_DOCSTRING,
)
class DistilBertForMaskedLM(DistilBertPreTrainedModel):
@@ -622,8 +620,9 @@ class DistilBertForMaskedLM(DistilBertPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -719,8 +718,9 @@ class DistilBertForSequenceClassification(DistilBertPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -836,12 +836,12 @@ class DistilBertForQuestionAnswering(DistilBertPreTrainedModel):
r"""
start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the start of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the end of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1052,7 +1052,8 @@ class DistilBertForMultipleChoice(DistilBertPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
+ Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
+ num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
`input_ids` above)
Returns:
diff --git a/src/transformers/models/distilbert/modeling_flax_distilbert.py b/src/transformers/models/distilbert/modeling_flax_distilbert.py
index b61453f918..da6073030a 100644
--- a/src/transformers/models/distilbert/modeling_flax_distilbert.py
+++ b/src/transformers/models/distilbert/modeling_flax_distilbert.py
@@ -47,12 +47,12 @@ _TOKENIZER_FOR_DOC = "DistilBertTokenizer"
FLAX_DISTILBERT_START_DOCSTRING = r"""
- This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading, saving and converting weights from
- PyTorch models)
+ This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading, saving and converting weights from PyTorch models)
- This model is also a Flax Linen [flax.linen.Module](https://flax.readthedocs.io/en/latest/flax.linen.html#module) subclass. Use it as a regular Flax linen Module
- and refer to the Flax documentation for all matter related to general usage and behavior.
+ This model is also a Flax Linen [flax.linen.Module](https://flax.readthedocs.io/en/latest/flax.linen.html#module)
+ subclass. Use it as a regular Flax linen Module and refer to the Flax documentation for all matter related to
+ general usage and behavior.
Finally, this model supports inherent JAX features such as:
@@ -64,8 +64,7 @@ FLAX_DISTILBERT_START_DOCSTRING = r"""
Parameters:
config ([`DistilBertConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
DISTILBERT_INPUTS_DOCSTRING = r"""
@@ -73,9 +72,8 @@ DISTILBERT_INPUTS_DOCSTRING = r"""
input_ids (`numpy.ndarray` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`BertTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`BertTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`numpy.ndarray` of shape `({0})`, *optional*):
@@ -598,7 +596,7 @@ class FlaxDistilBertForMaskedLMModule(nn.Module):
)
-@add_start_docstrings("""DistilBert Model with a `language modeling` head on top. """, FLAX_DISTILBERT_START_DOCSTRING)
+@add_start_docstrings("""DistilBert Model with a `language modeling` head on top.""", FLAX_DISTILBERT_START_DOCSTRING)
class FlaxDistilBertForMaskedLM(FlaxDistilBertPreTrainedModel):
module_class = FlaxDistilBertForMaskedLMModule
diff --git a/src/transformers/models/distilbert/modeling_tf_distilbert.py b/src/transformers/models/distilbert/modeling_tf_distilbert.py
index ec888623ef..a4b06d0c4d 100644
--- a/src/transformers/models/distilbert/modeling_tf_distilbert.py
+++ b/src/transformers/models/distilbert/modeling_tf_distilbert.py
@@ -451,13 +451,13 @@ class TFDistilBertPreTrainedModel(TFPreTrainedModel):
DISTILBERT_START_DOCSTRING = r"""
- This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading or saving, resizing the input
- embeddings, pruning heads etc.)
+ This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use
- it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage
- and behavior.
+ This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it
+ as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and
+ behavior.
@@ -466,11 +466,11 @@ DISTILBERT_START_DOCSTRING = r"""
- having all inputs as keyword arguments (like PyTorch models), or
- having all inputs as a list, tuple or dict in the first positional arguments.
- This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all
- the tensors in the first argument of the model call function: `model(inputs)`.
+ This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all the
+ tensors in the first argument of the model call function: `model(inputs)`.
- If you choose this second option, there are three possibilities you can use to gather all the input Tensors in
- the first positional argument :
+ If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the
+ first positional argument :
- a single Tensor with `input_ids` only and nothing else: `model(inputs_ids)`
- a list of varying length with one or several input Tensors IN THE ORDER given in the docstring:
@@ -483,8 +483,7 @@ DISTILBERT_START_DOCSTRING = r"""
Parameters:
config ([`DistilBertConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
DISTILBERT_INPUTS_DOCSTRING = r"""
@@ -492,9 +491,8 @@ DISTILBERT_INPUTS_DOCSTRING = r"""
input_ids (`Numpy array` or `tf.Tensor` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`DistilBertTokenizer`]. See
- [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for
- details.
+ Indices can be obtained using [`DistilBertTokenizer`]. See [`PreTrainedTokenizer.__call__`] and
+ [`PreTrainedTokenizer.encode`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`Numpy array` or `tf.Tensor` of shape `({0})`, *optional*):
@@ -511,9 +509,9 @@ DISTILBERT_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
inputs_embeds (`tf.Tensor` of shape `({0}, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the
@@ -523,8 +521,8 @@ DISTILBERT_INPUTS_DOCSTRING = r"""
more detail. This argument can be used only in eager mode, in graph mode the value in the config will be
used instead.
return_dict (`bool`, *optional*):
- Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This
- argument can be used in eager mode, in graph mode the value will always be set to True.
+ Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This argument can be used
+ in eager mode, in graph mode the value will always be set to True.
training (`bool`, *optional*, defaults to `False`):
Whether or not to use the model in training mode (some modules like dropout modules have different
behaviors between training and evaluation).
@@ -632,7 +630,7 @@ class TFDistilBertLMHead(tf.keras.layers.Layer):
@add_start_docstrings(
- """DistilBert Model with a `masked language modeling` head on top. """,
+ """DistilBert Model with a `masked language modeling` head on top.""",
DISTILBERT_START_DOCSTRING,
)
class TFDistilBertForMaskedLM(TFDistilBertPreTrainedModel, TFMaskedLanguageModelingLoss):
@@ -677,8 +675,9 @@ class TFDistilBertForMaskedLM(TFDistilBertPreTrainedModel, TFMaskedLanguageModel
):
r"""
labels (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
"""
inputs = input_processing(
func=self.call,
@@ -777,8 +776,9 @@ class TFDistilBertForSequenceClassification(TFDistilBertPreTrainedModel, TFSeque
):
r"""
labels (`tf.Tensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
inputs = input_processing(
func=self.call,
@@ -978,8 +978,8 @@ class TFDistilBertForMultipleChoice(TFDistilBertPreTrainedModel, TFMultipleChoic
):
r"""
labels (`tf.Tensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices]` where `num_choices` is the size of the second dimension of the input tensors. (See
- `input_ids` above)
+ Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices]`
+ where `num_choices` is the size of the second dimension of the input tensors. (See `input_ids` above)
"""
inputs = input_processing(
func=self.call,
@@ -1105,12 +1105,12 @@ class TFDistilBertForQuestionAnswering(TFDistilBertPreTrainedModel, TFQuestionAn
r"""
start_positions (`tf.Tensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the start of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
end_positions (`tf.Tensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the end of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
"""
inputs = input_processing(
func=self.call,
diff --git a/src/transformers/models/distilbert/tokenization_distilbert.py b/src/transformers/models/distilbert/tokenization_distilbert.py
index a04c1059c7..694c0ad25a 100644
--- a/src/transformers/models/distilbert/tokenization_distilbert.py
+++ b/src/transformers/models/distilbert/tokenization_distilbert.py
@@ -57,11 +57,10 @@ class DistilBertTokenizer(BertTokenizer):
r"""
Construct a DistilBERT tokenizer.
- [`DistilBertTokenizer`] is identical to [`BertTokenizer`] and runs end-to-end
- tokenization: punctuation splitting and wordpiece.
+ [`DistilBertTokenizer`] is identical to [`BertTokenizer`] and runs end-to-end tokenization: punctuation splitting
+ and wordpiece.
- Refer to superclass [`BertTokenizer`] for usage examples and documentation concerning
- parameters.
+ Refer to superclass [`BertTokenizer`] for usage examples and documentation concerning parameters.
"""
vocab_files_names = VOCAB_FILES_NAMES
diff --git a/src/transformers/models/distilbert/tokenization_distilbert_fast.py b/src/transformers/models/distilbert/tokenization_distilbert_fast.py
index 3b052f5cef..6a4ddfb819 100644
--- a/src/transformers/models/distilbert/tokenization_distilbert_fast.py
+++ b/src/transformers/models/distilbert/tokenization_distilbert_fast.py
@@ -66,11 +66,10 @@ class DistilBertTokenizerFast(BertTokenizerFast):
r"""
Construct a "fast" DistilBERT tokenizer (backed by HuggingFace's *tokenizers* library).
- [`DistilBertTokenizerFast`] is identical to [`BertTokenizerFast`] and runs
- end-to-end tokenization: punctuation splitting and wordpiece.
+ [`DistilBertTokenizerFast`] is identical to [`BertTokenizerFast`] and runs end-to-end tokenization: punctuation
+ splitting and wordpiece.
- Refer to superclass [`BertTokenizerFast`] for usage examples and documentation concerning
- parameters.
+ Refer to superclass [`BertTokenizerFast`] for usage examples and documentation concerning parameters.
"""
vocab_files_names = VOCAB_FILES_NAMES
diff --git a/src/transformers/models/dpr/configuration_dpr.py b/src/transformers/models/dpr/configuration_dpr.py
index dd0a9dfddc..973754854d 100644
--- a/src/transformers/models/dpr/configuration_dpr.py
+++ b/src/transformers/models/dpr/configuration_dpr.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" DPR model configuration """
+""" DPR model configuration"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging
@@ -34,12 +34,10 @@ class DPRConfig(PretrainedConfig):
r"""
[`DPRConfig`] is the configuration class to store the configuration of a *DPRModel*.
- This is the configuration class to store the configuration of a [`DPRContextEncoder`],
- [`DPRQuestionEncoder`], or a [`DPRReader`]. It is used to instantiate the
- components of the DPR model.
+ This is the configuration class to store the configuration of a [`DPRContextEncoder`], [`DPRQuestionEncoder`], or a
+ [`DPRReader`]. It is used to instantiate the components of the DPR model.
- This class is a subclass of [`BertConfig`]. Please check the superclass for the documentation of
- all kwargs.
+ This class is a subclass of [`BertConfig`]. Please check the superclass for the documentation of all kwargs.
Args:
vocab_size (`int`, *optional*, defaults to 30522):
@@ -54,8 +52,8 @@ class DPRConfig(PretrainedConfig):
intermediate_size (`int`, *optional*, defaults to 3072):
Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"silu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"silu"` and `"gelu_new"` are supported.
hidden_dropout_prob (`float`, *optional*, defaults to 0.1):
The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
attention_probs_dropout_prob (`float`, *optional*, defaults to 0.1):
@@ -70,10 +68,11 @@ class DPRConfig(PretrainedConfig):
layer_norm_eps (`float`, *optional*, defaults to 1e-12):
The epsilon used by the layer normalization layers.
position_embedding_type (`str`, *optional*, defaults to `"absolute"`):
- Type of position embedding. Choose one of `"absolute"`, `"relative_key"`,
- `"relative_key_query"`. For positional embeddings use `"absolute"`. For more information on
- `"relative_key"`, please refer to [Self-Attention with Relative Position Representations (Shaw et al.)](https://arxiv.org/abs/1803.02155). For more information on `"relative_key_query"`, please refer to
- *Method 4* in [Improve Transformer Models with Better Relative Position Embeddings (Huang et al.)](https://arxiv.org/abs/2009.13658).
+ Type of position embedding. Choose one of `"absolute"`, `"relative_key"`, `"relative_key_query"`. For
+ positional embeddings use `"absolute"`. For more information on `"relative_key"`, please refer to
+ [Self-Attention with Relative Position Representations (Shaw et al.)](https://arxiv.org/abs/1803.02155).
+ For more information on `"relative_key_query"`, please refer to *Method 4* in [Improve Transformer Models
+ with Better Relative Position Embeddings (Huang et al.)](https://arxiv.org/abs/2009.13658).
projection_dim (`int`, *optional*, defaults to 0):
Dimension of the projection for the context and question encoders. If it is set to zero (default), then no
projection is done.
diff --git a/src/transformers/models/dpr/modeling_dpr.py b/src/transformers/models/dpr/modeling_dpr.py
index 333106e4fb..aa2ccd4ce2 100644
--- a/src/transformers/models/dpr/modeling_dpr.py
+++ b/src/transformers/models/dpr/modeling_dpr.py
@@ -69,12 +69,13 @@ class DPRContextEncoderOutput(ModelOutput):
hidden-state of the first token of the sequence (classification token) further processed by a Linear layer.
This output is to be used to embed contexts for nearest neighbors queries with questions embeddings.
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -96,12 +97,13 @@ class DPRQuestionEncoderOutput(ModelOutput):
hidden-state of the first token of the sequence (classification token) further processed by a Linear layer.
This output is to be used to embed questions for nearest neighbors queries with context embeddings.
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -126,12 +128,13 @@ class DPRReaderOutput(ModelOutput):
Outputs of the QA classifier of the DPRReader that corresponds to the scores of each passage to answer the
question, compared to all the other passages.
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -327,19 +330,18 @@ class DPRPretrainedReader(DPRPreTrainedModel):
DPR_START_DOCSTRING = r"""
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
Parameters:
config ([`DPRConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
DPR_ENCODERS_INPUTS_DOCSTRING = r"""
@@ -365,9 +367,8 @@ DPR_ENCODERS_INPUTS_DOCSTRING = r"""
DPR is a model with absolute position embeddings so it's usually advised to pad the inputs on the right
rather than the left.
- Indices can be obtained using [`DPRTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`DPRTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -378,16 +379,17 @@ DPR_ENCODERS_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -402,16 +404,15 @@ DPR_READER_INPUTS_DOCSTRING = r"""
Args:
input_ids: (`Tuple[torch.LongTensor]` of shapes `(n_passages, sequence_length)`):
Indices of input sequence tokens in the vocabulary. It has to be a sequence triplet with 1) the question
- and 2) the passages titles and 3) the passages texts To match pretraining, DPR `input_ids` sequence
- should be formatted with [CLS] and [SEP] with the format:
+ and 2) the passages titles and 3) the passages texts To match pretraining, DPR `input_ids` sequence should
+ be formatted with [CLS] and [SEP] with the format:
`[CLS] [SEP] [SEP] `
DPR is a model with absolute position embeddings so it's usually advised to pad the inputs on the right
rather than the left.
- Indices can be obtained using [`DPRReaderTokenizer`]. See this class documentation for
- more details.
+ Indices can be obtained using [`DPRReaderTokenizer`]. See this class documentation for more details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.FloatTensor` of shape `(n_passages, sequence_length)`, *optional*):
@@ -422,9 +423,9 @@ DPR_READER_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
inputs_embeds (`torch.FloatTensor` of shape `(n_passages, sequence_length, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
diff --git a/src/transformers/models/dpr/modeling_tf_dpr.py b/src/transformers/models/dpr/modeling_tf_dpr.py
index 75e925cb2a..d48be40c38 100644
--- a/src/transformers/models/dpr/modeling_tf_dpr.py
+++ b/src/transformers/models/dpr/modeling_tf_dpr.py
@@ -66,12 +66,13 @@ class TFDPRContextEncoderOutput(ModelOutput):
hidden-state of the first token of the sequence (classification token) further processed by a Linear layer.
This output is to be used to embed contexts for nearest neighbors queries with questions embeddings.
hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -93,12 +94,13 @@ class TFDPRQuestionEncoderOutput(ModelOutput):
hidden-state of the first token of the sequence (classification token) further processed by a Linear layer.
This output is to be used to embed questions for nearest neighbors queries with context embeddings.
hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -123,12 +125,13 @@ class TFDPRReaderOutput(ModelOutput):
Outputs of the QA classifier of the DPRReader that corresponds to the scores of each passage to answer the
question, compared to all the other passages.
hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -441,9 +444,9 @@ class TFDPRPretrainedReader(TFPreTrainedModel):
TF_DPR_START_DOCSTRING = r"""
- This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading or saving, resizing the input
- embeddings, pruning heads etc.)
+ This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
This model is also a Tensorflow [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model)
subclass. Use it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to
@@ -456,11 +459,11 @@ TF_DPR_START_DOCSTRING = r"""
- having all inputs as keyword arguments (like PyTorch models), or
- having all inputs as a list, tuple or dict in the first positional arguments.
- This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all
- the tensors in the first argument of the model call function: `model(inputs)`.
+ This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all the
+ tensors in the first argument of the model call function: `model(inputs)`.
- If you choose this second option, there are three possibilities you can use to gather all the input Tensors in
- the first positional argument :
+ If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the
+ first positional argument :
- a single Tensor with `input_ids` only and nothing else: `model(inputs_ids)`
- a list of varying length with one or several input Tensors IN THE ORDER given in the docstring:
@@ -473,8 +476,7 @@ TF_DPR_START_DOCSTRING = r"""
Parameters:
config ([`DPRConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~TFPreTrainedModel.from_pretrained`] method to load the
- model weights.
+ configuration. Check out the [`~TFPreTrainedModel.from_pretrained`] method to load the model weights.
"""
TF_DPR_ENCODERS_INPUTS_DOCSTRING = r"""
@@ -500,9 +502,8 @@ TF_DPR_ENCODERS_INPUTS_DOCSTRING = r"""
DPR is a model with absolute position embeddings so it's usually advised to pad the inputs on the right
rather than the left.
- Indices can be obtained using [`DPRTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`DPRTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`Numpy array` or `tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -513,16 +514,17 @@ TF_DPR_ENCODERS_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`Numpy array` or `tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
inputs_embeds (`Numpy array` or `tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the
@@ -532,8 +534,8 @@ TF_DPR_ENCODERS_INPUTS_DOCSTRING = r"""
more detail. This argument can be used only in eager mode, in graph mode the value in the config will be
used instead.
return_dict (`bool`, *optional*):
- Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This
- argument can be used in eager mode, in graph mode the value will always be set to True.
+ Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This argument can be used
+ in eager mode, in graph mode the value will always be set to True.
training (`bool`, *optional*, defaults to `False`):
Whether or not to use the model in training mode (some modules like dropout modules have different
behaviors between training and evaluation).
@@ -543,16 +545,15 @@ TF_DPR_READER_INPUTS_DOCSTRING = r"""
Args:
input_ids: (`Numpy array` or `tf.Tensor` of shapes `(n_passages, sequence_length)`):
Indices of input sequence tokens in the vocabulary. It has to be a sequence triplet with 1) the question
- and 2) the passages titles and 3) the passages texts To match pretraining, DPR `input_ids` sequence
- should be formatted with [CLS] and [SEP] with the format:
+ and 2) the passages titles and 3) the passages texts To match pretraining, DPR `input_ids` sequence should
+ be formatted with [CLS] and [SEP] with the format:
`[CLS] [SEP] [SEP] `
DPR is a model with absolute position embeddings so it's usually advised to pad the inputs on the right
rather than the left.
- Indices can be obtained using [`DPRReaderTokenizer`]. See this class documentation for
- more details.
+ Indices can be obtained using [`DPRReaderTokenizer`]. See this class documentation for more details.
attention_mask (`Numpy array` or `tf.Tensor` of shape `(n_passages, sequence_length)`, *optional*):
Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
@@ -561,16 +562,16 @@ TF_DPR_READER_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
inputs_embeds (`Numpy array` or `tf.Tensor` of shape `(n_passages, sequence_length, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_hidden_states (`bool`, *optional*):
Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
more detail. This argument can be used only in eager mode, in graph mode the value in the config will be
used instead.
return_dict (`bool`, *optional*):
- Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This
- argument can be used in eager mode, in graph mode the value will always be set to True.
+ Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This argument can be used
+ in eager mode, in graph mode the value will always be set to True.
training (`bool`, *optional*, defaults to `False`):
Whether or not to use the model in training mode (some modules like dropout modules have different
behaviors between training and evaluation).
diff --git a/src/transformers/models/dpr/tokenization_dpr.py b/src/transformers/models/dpr/tokenization_dpr.py
index 46ab974262..d5e3c29732 100644
--- a/src/transformers/models/dpr/tokenization_dpr.py
+++ b/src/transformers/models/dpr/tokenization_dpr.py
@@ -91,11 +91,10 @@ class DPRContextEncoderTokenizer(BertTokenizer):
r"""
Construct a DPRContextEncoder tokenizer.
- [`DPRContextEncoderTokenizer`] is identical to [`BertTokenizer`] and runs
- end-to-end tokenization: punctuation splitting and wordpiece.
+ [`DPRContextEncoderTokenizer`] is identical to [`BertTokenizer`] and runs end-to-end tokenization: punctuation
+ splitting and wordpiece.
- Refer to superclass [`BertTokenizer`] for usage examples and documentation concerning
- parameters.
+ Refer to superclass [`BertTokenizer`] for usage examples and documentation concerning parameters.
"""
vocab_files_names = VOCAB_FILES_NAMES
@@ -108,11 +107,10 @@ class DPRQuestionEncoderTokenizer(BertTokenizer):
r"""
Constructs a DPRQuestionEncoder tokenizer.
- [`DPRQuestionEncoderTokenizer`] is identical to [`BertTokenizer`] and runs
- end-to-end tokenization: punctuation splitting and wordpiece.
+ [`DPRQuestionEncoderTokenizer`] is identical to [`BertTokenizer`] and runs end-to-end tokenization: punctuation
+ splitting and wordpiece.
- Refer to superclass [`BertTokenizer`] for usage examples and documentation concerning
- parameters.
+ Refer to superclass [`BertTokenizer`] for usage examples and documentation concerning parameters.
"""
vocab_files_names = VOCAB_FILES_NAMES
@@ -129,10 +127,10 @@ DPRReaderOutput = collections.namedtuple("DPRReaderOutput", ["start_logits", "en
CUSTOM_DPR_READER_DOCSTRING = r"""
- Return a dictionary with the token ids of the input strings and other information to give to
- `.decode_best_spans`. It converts the strings of a question and different passages (title and text) in a
- sequence of IDs (integers), using the tokenizer and vocabulary. The resulting `input_ids` is a matrix of size
- `(n_passages, sequence_length)` with the format:
+ Return a dictionary with the token ids of the input strings and other information to give to `.decode_best_spans`.
+ It converts the strings of a question and different passages (title and text) in a sequence of IDs (integers),
+ using the tokenizer and vocabulary. The resulting `input_ids` is a matrix of size `(n_passages, sequence_length)`
+ with the format:
```
[CLS] [SEP] [SEP]
@@ -141,8 +139,8 @@ CUSTOM_DPR_READER_DOCSTRING = r"""
Args:
questions (`str` or `List[str]`):
The questions to be encoded. You can specify one question for many passages. In this case, the question
- will be duplicated like `[questions] * n_passages`. Otherwise you have to specify as many questions as
- in `titles` or `texts`.
+ will be duplicated like `[questions] * n_passages`. Otherwise you have to specify as many questions as in
+ `titles` or `texts`.
titles (`str` or `List[str]`):
The passages titles to be encoded. This can be a string or a list of strings if there are several passages.
texts (`str` or `List[str]`):
@@ -150,33 +148,33 @@ CUSTOM_DPR_READER_DOCSTRING = r"""
padding (`bool`, `str` or [`~file_utils.PaddingStrategy`], *optional*, defaults to `False`):
Activates and controls padding. Accepts the following values:
- - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
- sequence if provided).
- - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the
- maximum acceptable input length for the model if that argument is not provided.
- - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of
- different lengths).
+ - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single sequence
+ if provided).
+ - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
+ acceptable input length for the model if that argument is not provided.
+ - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different
+ lengths).
truncation (`bool`, `str` or [`~tokenization_utils_base.TruncationStrategy`], *optional*, defaults to `False`):
Activates and controls truncation. Accepts the following values:
- - `True` or `'longest_first'`: Truncate to a maximum length specified with the argument
- `max_length` or to the maximum acceptable input length for the model if that argument is not
- provided. This will truncate token by token, removing a token from the longest sequence in the pair if a
- pair of sequences (or a batch of pairs) is provided.
- - `'only_first'`: Truncate to a maximum length specified with the argument `max_length` or to the
- maximum acceptable input length for the model if that argument is not provided. This will only truncate
- the first sequence of a pair if a pair of sequences (or a batch of pairs) is provided.
- - `'only_second'`: Truncate to a maximum length specified with the argument `max_length` or to
- the maximum acceptable input length for the model if that argument is not provided. This will only
- truncate the second sequence of a pair if a pair of sequences (or a batch of pairs) is provided.
- - `False` or `'do_not_truncate'` (default): No truncation (i.e., can output batch with sequence
- lengths greater than the model maximum admissible input size).
+ - `True` or `'longest_first'`: Truncate to a maximum length specified with the argument `max_length` or to
+ the maximum acceptable input length for the model if that argument is not provided. This will truncate
+ token by token, removing a token from the longest sequence in the pair if a pair of sequences (or a batch
+ of pairs) is provided.
+ - `'only_first'`: Truncate to a maximum length specified with the argument `max_length` or to the maximum
+ acceptable input length for the model if that argument is not provided. This will only truncate the first
+ sequence of a pair if a pair of sequences (or a batch of pairs) is provided.
+ - `'only_second'`: Truncate to a maximum length specified with the argument `max_length` or to the maximum
+ acceptable input length for the model if that argument is not provided. This will only truncate the
+ second sequence of a pair if a pair of sequences (or a batch of pairs) is provided.
+ - `False` or `'do_not_truncate'` (default): No truncation (i.e., can output batch with sequence lengths
+ greater than the model maximum admissible input size).
max_length (`int`, *optional*):
Controls the maximum length to use by one of the truncation/padding parameters.
- If left unset or set to `None`, this will use the predefined model maximum length if a maximum
- length is required by one of the truncation/padding parameters. If the model has no specific maximum
- input length (like XLNet) truncation/padding to a maximum length will be deactivated.
+ If left unset or set to `None`, this will use the predefined model maximum length if a maximum length
+ is required by one of the truncation/padding parameters. If the model has no specific maximum input
+ length (like XLNet) truncation/padding to a maximum length will be deactivated.
return_tensors (`str` or [`~file_utils.TensorType`], *optional*):
If set, will return tensors instead of list of python integers. Acceptable values are:
@@ -271,11 +269,12 @@ class CustomDPRReaderTokenizerMixin:
Returns: *List* of *DPRReaderOutput* sorted by descending *(relevance_score, span_score)*. Each
*DPRReaderOutput* is a *Tuple* with:
- - **span_score**: `float` that corresponds to the score given by the reader for this span compared to
- other spans in the same passage. It corresponds to the sum of the start and end logits of the span.
+ - **span_score**: `float` that corresponds to the score given by the reader for this span compared to other
+ spans in the same passage. It corresponds to the sum of the start and end logits of the span.
- **relevance_score**: `float` that corresponds to the score of the each passage to answer the question,
compared to all the other passages. It corresponds to the output of the QA classifier of the DPRReader.
- - **doc_id**: ``int``` the id of the passage. - **start_index**: `int` the start index of the span (inclusive). - **end_index**: `int` the end index of the span (inclusive).
+ - **doc_id**: ``int``` the id of the passage. - **start_index**: `int` the start index of the span
+ (inclusive). - **end_index**: `int` the end index of the span (inclusive).
Examples:
@@ -371,12 +370,11 @@ class DPRReaderTokenizer(CustomDPRReaderTokenizerMixin, BertTokenizer):
r"""
Construct a DPRReader tokenizer.
- [`DPRReaderTokenizer`] is almost identical to [`BertTokenizer`] and runs
- end-to-end tokenization: punctuation splitting and wordpiece. The difference is that is has three inputs strings:
- question, titles and texts that are combined to be fed to the [`DPRReader`] model.
+ [`DPRReaderTokenizer`] is almost identical to [`BertTokenizer`] and runs end-to-end tokenization: punctuation
+ splitting and wordpiece. The difference is that is has three inputs strings: question, titles and texts that are
+ combined to be fed to the [`DPRReader`] model.
- Refer to superclass [`BertTokenizer`] for usage examples and documentation concerning
- parameters.
+ Refer to superclass [`BertTokenizer`] for usage examples and documentation concerning parameters.
"""
vocab_files_names = VOCAB_FILES_NAMES
diff --git a/src/transformers/models/dpr/tokenization_dpr_fast.py b/src/transformers/models/dpr/tokenization_dpr_fast.py
index d4d5f41418..6be8087371 100644
--- a/src/transformers/models/dpr/tokenization_dpr_fast.py
+++ b/src/transformers/models/dpr/tokenization_dpr_fast.py
@@ -92,11 +92,10 @@ class DPRContextEncoderTokenizerFast(BertTokenizerFast):
r"""
Construct a "fast" DPRContextEncoder tokenizer (backed by HuggingFace's *tokenizers* library).
- [`DPRContextEncoderTokenizerFast`] is identical to [`BertTokenizerFast`] and
- runs end-to-end tokenization: punctuation splitting and wordpiece.
+ [`DPRContextEncoderTokenizerFast`] is identical to [`BertTokenizerFast`] and runs end-to-end tokenization:
+ punctuation splitting and wordpiece.
- Refer to superclass [`BertTokenizerFast`] for usage examples and documentation concerning
- parameters.
+ Refer to superclass [`BertTokenizerFast`] for usage examples and documentation concerning parameters.
"""
vocab_files_names = VOCAB_FILES_NAMES
@@ -110,11 +109,10 @@ class DPRQuestionEncoderTokenizerFast(BertTokenizerFast):
r"""
Constructs a "fast" DPRQuestionEncoder tokenizer (backed by HuggingFace's *tokenizers* library).
- [`DPRQuestionEncoderTokenizerFast`] is identical to [`BertTokenizerFast`] and
- runs end-to-end tokenization: punctuation splitting and wordpiece.
+ [`DPRQuestionEncoderTokenizerFast`] is identical to [`BertTokenizerFast`] and runs end-to-end tokenization:
+ punctuation splitting and wordpiece.
- Refer to superclass [`BertTokenizerFast`] for usage examples and documentation concerning
- parameters.
+ Refer to superclass [`BertTokenizerFast`] for usage examples and documentation concerning parameters.
"""
vocab_files_names = VOCAB_FILES_NAMES
@@ -132,18 +130,18 @@ DPRReaderOutput = collections.namedtuple("DPRReaderOutput", ["start_logits", "en
CUSTOM_DPR_READER_DOCSTRING = r"""
- Return a dictionary with the token ids of the input strings and other information to give to
- `.decode_best_spans`. It converts the strings of a question and different passages (title and text) in a
- sequence of IDs (integers), using the tokenizer and vocabulary. The resulting `input_ids` is a matrix of size
- `(n_passages, sequence_length)` with the format:
+ Return a dictionary with the token ids of the input strings and other information to give to `.decode_best_spans`.
+ It converts the strings of a question and different passages (title and text) in a sequence of IDs (integers),
+ using the tokenizer and vocabulary. The resulting `input_ids` is a matrix of size `(n_passages, sequence_length)`
+ with the format:
[CLS] [SEP] [SEP]
Args:
questions (`str` or `List[str]`):
The questions to be encoded. You can specify one question for many passages. In this case, the question
- will be duplicated like `[questions] * n_passages`. Otherwise you have to specify as many questions as
- in `titles` or `texts`.
+ will be duplicated like `[questions] * n_passages`. Otherwise you have to specify as many questions as in
+ `titles` or `texts`.
titles (`str` or `List[str]`):
The passages titles to be encoded. This can be a string or a list of strings if there are several passages.
texts (`str` or `List[str]`):
@@ -151,33 +149,33 @@ CUSTOM_DPR_READER_DOCSTRING = r"""
padding (`bool`, `str` or [`~file_utils.PaddingStrategy`], *optional*, defaults to `False`):
Activates and controls padding. Accepts the following values:
- - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
- sequence if provided).
- - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the
- maximum acceptable input length for the model if that argument is not provided.
- - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of
- different lengths).
+ - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single sequence
+ if provided).
+ - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
+ acceptable input length for the model if that argument is not provided.
+ - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different
+ lengths).
truncation (`bool`, `str` or [`~tokenization_utils_base.TruncationStrategy`], *optional*, defaults to `False`):
Activates and controls truncation. Accepts the following values:
- - `True` or `'longest_first'`: Truncate to a maximum length specified with the argument
- `max_length` or to the maximum acceptable input length for the model if that argument is not
- provided. This will truncate token by token, removing a token from the longest sequence in the pair if a
- pair of sequences (or a batch of pairs) is provided.
- - `'only_first'`: Truncate to a maximum length specified with the argument `max_length` or to the
- maximum acceptable input length for the model if that argument is not provided. This will only truncate
- the first sequence of a pair if a pair of sequences (or a batch of pairs) is provided.
- - `'only_second'`: Truncate to a maximum length specified with the argument `max_length` or to
- the maximum acceptable input length for the model if that argument is not provided. This will only
- truncate the second sequence of a pair if a pair of sequences (or a batch of pairs) is provided.
- - `False` or `'do_not_truncate'` (default): No truncation (i.e., can output batch with sequence
- lengths greater than the model maximum admissible input size).
+ - `True` or `'longest_first'`: Truncate to a maximum length specified with the argument `max_length` or to
+ the maximum acceptable input length for the model if that argument is not provided. This will truncate
+ token by token, removing a token from the longest sequence in the pair if a pair of sequences (or a batch
+ of pairs) is provided.
+ - `'only_first'`: Truncate to a maximum length specified with the argument `max_length` or to the maximum
+ acceptable input length for the model if that argument is not provided. This will only truncate the first
+ sequence of a pair if a pair of sequences (or a batch of pairs) is provided.
+ - `'only_second'`: Truncate to a maximum length specified with the argument `max_length` or to the maximum
+ acceptable input length for the model if that argument is not provided. This will only truncate the
+ second sequence of a pair if a pair of sequences (or a batch of pairs) is provided.
+ - `False` or `'do_not_truncate'` (default): No truncation (i.e., can output batch with sequence lengths
+ greater than the model maximum admissible input size).
max_length (`int`, *optional*):
Controls the maximum length to use by one of the truncation/padding parameters.
- If left unset or set to `None`, this will use the predefined model maximum length if a maximum
- length is required by one of the truncation/padding parameters. If the model has no specific maximum
- input length (like XLNet) truncation/padding to a maximum length will be deactivated.
+ If left unset or set to `None`, this will use the predefined model maximum length if a maximum length
+ is required by one of the truncation/padding parameters. If the model has no specific maximum input
+ length (like XLNet) truncation/padding to a maximum length will be deactivated.
return_tensors (`str` or [`~file_utils.TensorType`], *optional*):
If set, will return tensors instead of list of python integers. Acceptable values are:
@@ -272,11 +270,12 @@ class CustomDPRReaderTokenizerMixin:
Returns: *List* of *DPRReaderOutput* sorted by descending *(relevance_score, span_score)*. Each
*DPRReaderOutput* is a *Tuple* with:
- - **span_score**: `float` that corresponds to the score given by the reader for this span compared to
- other spans in the same passage. It corresponds to the sum of the start and end logits of the span.
+ - **span_score**: `float` that corresponds to the score given by the reader for this span compared to other
+ spans in the same passage. It corresponds to the sum of the start and end logits of the span.
- **relevance_score**: `float` that corresponds to the score of the each passage to answer the question,
compared to all the other passages. It corresponds to the output of the QA classifier of the DPRReader.
- - **doc_id**: ``int``` the id of the passage. - ***start_index**: `int` the start index of the span (inclusive). - **end_index**: `int` the end index of the span (inclusive).
+ - **doc_id**: ``int``` the id of the passage. - ***start_index**: `int` the start index of the span
+ (inclusive). - **end_index**: `int` the end index of the span (inclusive).
Examples:
@@ -372,12 +371,11 @@ class DPRReaderTokenizerFast(CustomDPRReaderTokenizerMixin, BertTokenizerFast):
r"""
Constructs a "fast" DPRReader tokenizer (backed by HuggingFace's *tokenizers* library).
- [`DPRReaderTokenizerFast`] is almost identical to [`BertTokenizerFast`] and
- runs end-to-end tokenization: punctuation splitting and wordpiece. The difference is that is has three inputs
- strings: question, titles and texts that are combined to be fed to the [`DPRReader`] model.
+ [`DPRReaderTokenizerFast`] is almost identical to [`BertTokenizerFast`] and runs end-to-end tokenization:
+ punctuation splitting and wordpiece. The difference is that is has three inputs strings: question, titles and texts
+ that are combined to be fed to the [`DPRReader`] model.
- Refer to superclass [`BertTokenizerFast`] for usage examples and documentation concerning
- parameters.
+ Refer to superclass [`BertTokenizerFast`] for usage examples and documentation concerning parameters.
"""
diff --git a/src/transformers/models/electra/configuration_electra.py b/src/transformers/models/electra/configuration_electra.py
index 963460318e..43b3b3255d 100644
--- a/src/transformers/models/electra/configuration_electra.py
+++ b/src/transformers/models/electra/configuration_electra.py
@@ -13,7 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" ELECTRA model configuration """
+""" ELECTRA model configuration"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging
@@ -33,20 +33,19 @@ ELECTRA_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class ElectraConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`ElectraModel`] or a
- [`TFElectraModel`]. It is used to instantiate a ELECTRA model according to the specified
- arguments, defining the model architecture. Instantiating a configuration with the defaults will yield a similar
- configuration to that of the ELECTRA [google/electra-small-discriminator](https://huggingface.co/google/electra-small-discriminator) architecture.
+ This is the configuration class to store the configuration of a [`ElectraModel`] or a [`TFElectraModel`]. It is
+ used to instantiate a ELECTRA model according to the specified arguments, defining the model architecture.
+ Instantiating a configuration with the defaults will yield a similar configuration to that of the ELECTRA
+ [google/electra-small-discriminator](https://huggingface.co/google/electra-small-discriminator) architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
vocab_size (`int`, *optional*, defaults to 30522):
Vocabulary size of the ELECTRA model. Defines the number of different tokens that can be represented by the
- `inputs_ids` passed when calling [`ElectraModel`] or
- [`TFElectraModel`].
+ `inputs_ids` passed when calling [`ElectraModel`] or [`TFElectraModel`].
embedding_size (`int`, *optional*, defaults to 128):
Dimensionality of the encoder layers and the pooler layer.
hidden_size (`int`, *optional*, defaults to 256):
@@ -58,8 +57,8 @@ class ElectraConfig(PretrainedConfig):
intermediate_size (`int`, *optional*, defaults to 1024):
Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
hidden_act (`str` or `Callable`, *optional*, defaults to `"gelu"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"silu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"silu"` and `"gelu_new"` are supported.
hidden_dropout_prob (`float`, *optional*, defaults to 0.1):
The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
attention_probs_dropout_prob (`float`, *optional*, defaults to 0.1):
@@ -68,8 +67,7 @@ class ElectraConfig(PretrainedConfig):
The maximum sequence length that this model might ever be used with. Typically set this to something large
just in case (e.g., 512 or 1024 or 2048).
type_vocab_size (`int`, *optional*, defaults to 2):
- The vocabulary size of the `token_type_ids` passed when calling [`ElectraModel`] or
- [`TFElectraModel`].
+ The vocabulary size of the `token_type_ids` passed when calling [`ElectraModel`] or [`TFElectraModel`].
initializer_range (`float`, *optional*, defaults to 0.02):
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
layer_norm_eps (`float`, *optional*, defaults to 1e-12):
@@ -97,10 +95,11 @@ class ElectraConfig(PretrainedConfig):
The dropout ratio to be used after the projection and activation.
position_embedding_type (`str`, *optional*, defaults to `"absolute"`):
- Type of position embedding. Choose one of `"absolute"`, `"relative_key"`,
- `"relative_key_query"`. For positional embeddings use `"absolute"`. For more information on
- `"relative_key"`, please refer to [Self-Attention with Relative Position Representations (Shaw et al.)](https://arxiv.org/abs/1803.02155). For more information on `"relative_key_query"`, please refer to
- *Method 4* in [Improve Transformer Models with Better Relative Position Embeddings (Huang et al.)](https://arxiv.org/abs/2009.13658).
+ Type of position embedding. Choose one of `"absolute"`, `"relative_key"`, `"relative_key_query"`. For
+ positional embeddings use `"absolute"`. For more information on `"relative_key"`, please refer to
+ [Self-Attention with Relative Position Representations (Shaw et al.)](https://arxiv.org/abs/1803.02155).
+ For more information on `"relative_key_query"`, please refer to *Method 4* in [Improve Transformer Models
+ with Better Relative Position Embeddings (Huang et al.)](https://arxiv.org/abs/2009.13658).
use_cache (`bool`, *optional*, defaults to `True`):
Whether or not the model should return the last key/values attentions (not used by all models). Only
relevant if `config.is_decoder=True`.
diff --git a/src/transformers/models/electra/modeling_electra.py b/src/transformers/models/electra/modeling_electra.py
index 8c343e4c68..3f00111518 100644
--- a/src/transformers/models/electra/modeling_electra.py
+++ b/src/transformers/models/electra/modeling_electra.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-"""PyTorch ELECTRA model. """
+"""PyTorch ELECTRA model."""
import math
import os
@@ -704,12 +704,13 @@ class ElectraForPreTrainingOutput(ModelOutput):
logits (`torch.FloatTensor` of shape `(batch_size, sequence_length)`):
Prediction scores of the head (scores for each token before SoftMax).
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -723,19 +724,18 @@ class ElectraForPreTrainingOutput(ModelOutput):
ELECTRA_START_DOCSTRING = r"""
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
Parameters:
config ([`ElectraConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
ELECTRA_INPUTS_DOCSTRING = r"""
@@ -743,9 +743,8 @@ ELECTRA_INPUTS_DOCSTRING = r"""
input_ids (`torch.LongTensor` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`ElectraTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`ElectraTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
@@ -756,14 +755,16 @@ ELECTRA_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
@@ -773,9 +774,9 @@ ELECTRA_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
encoder_hidden_states (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if
the model is configured as a decoder.
@@ -990,8 +991,9 @@ class ElectraForSequenceClassification(ElectraPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1079,8 +1081,8 @@ class ElectraForPreTraining(ElectraPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the ELECTRA loss. Input should be a sequence of tokens (see `input_ids`
- docstring) Indices should be in `[0, 1]`:
+ Labels for computing the ELECTRA loss. Input should be a sequence of tokens (see `input_ids` docstring)
+ Indices should be in `[0, 1]`:
- 0 indicates the token is an original token,
- 1 indicates the token was replaced.
@@ -1187,8 +1189,9 @@ class ElectraForMaskedLM(ElectraPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1360,12 +1363,12 @@ class ElectraForQuestionAnswering(ElectraPreTrainedModel):
r"""
start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the start of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the end of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1460,7 +1463,8 @@ class ElectraForMultipleChoice(ElectraPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
+ Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
+ num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
`input_ids` above)
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1512,7 +1516,7 @@ class ElectraForMultipleChoice(ElectraPreTrainedModel):
@add_start_docstrings(
- """ELECTRA Model with a `language modeling` head on top for CLM fine-tuning. """, ELECTRA_START_DOCSTRING
+ """ELECTRA Model with a `language modeling` head on top for CLM fine-tuning.""", ELECTRA_START_DOCSTRING
)
class ElectraForCausalLM(ElectraPreTrainedModel):
def __init__(self, config):
@@ -1570,12 +1574,12 @@ class ElectraForCausalLM(ElectraPreTrainedModel):
past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ `decoder_input_ids` of shape `(batch_size, sequence_length)`.
use_cache (`bool`, *optional*):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`).
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`).
Returns:
diff --git a/src/transformers/models/electra/modeling_flax_electra.py b/src/transformers/models/electra/modeling_flax_electra.py
index afa028fb45..8ef3f270e0 100644
--- a/src/transformers/models/electra/modeling_flax_electra.py
+++ b/src/transformers/models/electra/modeling_flax_electra.py
@@ -62,12 +62,13 @@ class FlaxElectraForPreTrainingOutput(ModelOutput):
logits (`jnp.ndarray` of shape `(batch_size, sequence_length, config.vocab_size)`):
Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
hidden_states (`tuple(jnp.ndarray)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(jnp.ndarray)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -80,12 +81,12 @@ class FlaxElectraForPreTrainingOutput(ModelOutput):
ELECTRA_START_DOCSTRING = r"""
- This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading, saving and converting weights from
- PyTorch models)
+ This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading, saving and converting weights from PyTorch models)
- This model is also a Flax Linen [flax.nn.Module](https://flax.readthedocs.io/en/latest/_autosummary/flax.nn.module.html) subclass. Use it as a regular Flax
- Module and refer to the Flax documentation for all matter related to general usage and behavior.
+ This model is also a Flax Linen
+ [flax.nn.Module](https://flax.readthedocs.io/en/latest/_autosummary/flax.nn.module.html) subclass. Use it as a
+ regular Flax Module and refer to the Flax documentation for all matter related to general usage and behavior.
Finally, this model supports inherent JAX features such as:
@@ -97,8 +98,7 @@ ELECTRA_START_DOCSTRING = r"""
Parameters:
config ([`ElectraConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
ELECTRA_INPUTS_DOCSTRING = r"""
@@ -106,9 +106,8 @@ ELECTRA_INPUTS_DOCSTRING = r"""
input_ids (`numpy.ndarray` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`ElectraTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`ElectraTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`numpy.ndarray` of shape `({0})`, *optional*):
@@ -119,15 +118,18 @@ ELECTRA_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`numpy.ndarray` of shape `({0})`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`numpy.ndarray` of shape `({0})`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
- head_mask (`numpy.ndarray` of shape `({0})`, `optional): Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`:
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
+ head_mask (`numpy.ndarray` of shape `({0})`, `optional):
+ Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`:
- 1 indicates the head is **not masked**,
- 0 indicates the head is **masked**.
@@ -743,7 +745,7 @@ class FlaxElectraForMaskedLMModule(nn.Module):
)
-@add_start_docstrings("""Electra Model with a `language modeling` head on top. """, ELECTRA_START_DOCSTRING)
+@add_start_docstrings("""Electra Model with a `language modeling` head on top.""", ELECTRA_START_DOCSTRING)
class FlaxElectraForMaskedLM(FlaxElectraPreTrainedModel):
module_class = FlaxElectraForMaskedLMModule
@@ -926,14 +928,12 @@ class FlaxElectraSequenceSummary(nn.Module):
config class of your model for the default values it uses):
- **summary_use_proj** (`bool`) -- Add a projection after the vector extraction.
- - **summary_proj_to_labels** (`bool`) -- If `True`, the projection outputs to
- `config.num_labels` classes (otherwise to `config.hidden_size`).
- - **summary_activation** (`Optional[str]`) -- Set to `"tanh"` to add a tanh activation to the
- output, another string or `None` will add no activation.
- - **summary_first_dropout** (`float`) -- Optional dropout probability before the projection and
- activation.
- - **summary_last_dropout** (`float`)-- Optional dropout probability after the projection and
- activation.
+ - **summary_proj_to_labels** (`bool`) -- If `True`, the projection outputs to `config.num_labels` classes
+ (otherwise to `config.hidden_size`).
+ - **summary_activation** (`Optional[str]`) -- Set to `"tanh"` to add a tanh activation to the output,
+ another string or `None` will add no activation.
+ - **summary_first_dropout** (`float`) -- Optional dropout probability before the projection and activation.
+ - **summary_last_dropout** (`float`)-- Optional dropout probability after the projection and activation.
"""
config: ElectraConfig
dtype: jnp.dtype = jnp.float32
@@ -970,8 +970,7 @@ class FlaxElectraSequenceSummary(nn.Module):
hidden_states (`jnp.array` of shape `[batch_size, seq_len, hidden_size]`):
The hidden states of the last layer.
cls_index (`jnp.array` of shape `[batch_size]` or `[batch_size, ...]` where ... are optional leading dimensions of `hidden_states`, *optional*):
- Used if `summary_type == "cls_index"` and takes the last token of the sequence as classification
- token.
+ Used if `summary_type == "cls_index"` and takes the last token of the sequence as classification token.
Returns:
`jnp.array`: The summary of the sequence hidden states.
diff --git a/src/transformers/models/electra/modeling_tf_electra.py b/src/transformers/models/electra/modeling_tf_electra.py
index 2dedc146f6..8e734e24e9 100644
--- a/src/transformers/models/electra/modeling_tf_electra.py
+++ b/src/transformers/models/electra/modeling_tf_electra.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" TF Electra model. """
+""" TF Electra model."""
import math
import warnings
@@ -830,12 +830,13 @@ class TFElectraForPreTrainingOutput(ModelOutput):
logits (`tf.Tensor` of shape `(batch_size, sequence_length)`):
Prediction scores of the head (scores for each token before SoftMax).
hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -848,13 +849,13 @@ class TFElectraForPreTrainingOutput(ModelOutput):
ELECTRA_START_DOCSTRING = r"""
- This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading or saving, resizing the input
- embeddings, pruning heads etc.)
+ This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use
- it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage
- and behavior.
+ This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it
+ as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and
+ behavior.
@@ -863,11 +864,11 @@ ELECTRA_START_DOCSTRING = r"""
- having all inputs as keyword arguments (like PyTorch models), or
- having all inputs as a list, tuple or dict in the first positional arguments.
- This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all
- the tensors in the first argument of the model call function: `model(inputs)`.
+ This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all the
+ tensors in the first argument of the model call function: `model(inputs)`.
- If you choose this second option, there are three possibilities you can use to gather all the input Tensors in
- the first positional argument :
+ If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the
+ first positional argument :
- a single Tensor with `input_ids` only and nothing else: `model(inputs_ids)`
- a list of varying length with one or several input Tensors IN THE ORDER given in the docstring:
@@ -880,8 +881,7 @@ ELECTRA_START_DOCSTRING = r"""
Parameters:
config ([`ElectraConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
ELECTRA_INPUTS_DOCSTRING = r"""
@@ -889,9 +889,8 @@ ELECTRA_INPUTS_DOCSTRING = r"""
input_ids (`Numpy array` or `tf.Tensor` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`ElectraTokenizer`]. See
- [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for
- details.
+ Indices can be obtained using [`ElectraTokenizer`]. See [`PreTrainedTokenizer.__call__`] and
+ [`PreTrainedTokenizer.encode`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`Numpy array` or `tf.Tensor` of shape `({0})`, *optional*):
@@ -902,7 +901,8 @@ ELECTRA_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
position_ids (`Numpy array` or `tf.Tensor` of shape `({0})`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
head_mask (`Numpy array` or `tf.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
@@ -912,9 +912,9 @@ ELECTRA_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
inputs_embeds (`tf.Tensor` of shape `({0}, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the
@@ -924,8 +924,8 @@ ELECTRA_INPUTS_DOCSTRING = r"""
more detail. This argument can be used only in eager mode, in graph mode the value in the config will be
used instead.
return_dict (`bool`, *optional*):
- Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This
- argument can be used in eager mode, in graph mode the value will always be set to True.
+ Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This argument can be used
+ in eager mode, in graph mode the value will always be set to True.
training (`bool`, *optional*, defaults to `False`):
Whether or not to use the model in training mode (some modules like dropout modules have different
behaviors between training and evaluation).
@@ -984,12 +984,12 @@ class TFElectraModel(TFElectraPreTrainedModel):
past_key_values (`Tuple[Tuple[tf.Tensor]]` of length `config.n_layers`)
contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ `decoder_input_ids` of shape `(batch_size, sequence_length)`.
use_cache (`bool`, *optional*, defaults to `True`):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`). Set to `False` during training, `True` during generation
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`). Set to `False` during training, `True` during generation
"""
inputs = input_processing(
func=self.call,
@@ -1232,8 +1232,9 @@ class TFElectraForMaskedLM(TFElectraPreTrainedModel, TFMaskedLanguageModelingLos
):
r"""
labels (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
"""
inputs = input_processing(
func=self.call,
@@ -1356,8 +1357,9 @@ class TFElectraForSequenceClassification(TFElectraPreTrainedModel, TFSequenceCla
):
r"""
labels (`tf.Tensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
inputs = input_processing(
func=self.call,
@@ -1463,8 +1465,8 @@ class TFElectraForMultipleChoice(TFElectraPreTrainedModel, TFMultipleChoiceLoss)
):
r"""
labels (`tf.Tensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices]` where `num_choices` is the size of the second dimension of the input tensors. (See
- `input_ids` above)
+ Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices]`
+ where `num_choices` is the size of the second dimension of the input tensors. (See `input_ids` above)
"""
inputs = input_processing(
func=self.call,
@@ -1700,12 +1702,12 @@ class TFElectraForQuestionAnswering(TFElectraPreTrainedModel, TFQuestionAnswerin
r"""
start_positions (`tf.Tensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the start of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
end_positions (`tf.Tensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the end of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
"""
inputs = input_processing(
func=self.call,
diff --git a/src/transformers/models/electra/tokenization_electra.py b/src/transformers/models/electra/tokenization_electra.py
index 8f087263d8..9fd5568cde 100644
--- a/src/transformers/models/electra/tokenization_electra.py
+++ b/src/transformers/models/electra/tokenization_electra.py
@@ -53,11 +53,10 @@ class ElectraTokenizer(BertTokenizer):
r"""
Construct an ELECTRA tokenizer.
- [`ElectraTokenizer`] is identical to [`BertTokenizer`] and runs end-to-end
- tokenization: punctuation splitting and wordpiece.
+ [`ElectraTokenizer`] is identical to [`BertTokenizer`] and runs end-to-end tokenization: punctuation splitting and
+ wordpiece.
- Refer to superclass [`BertTokenizer`] for usage examples and documentation concerning
- parameters.
+ Refer to superclass [`BertTokenizer`] for usage examples and documentation concerning parameters.
"""
vocab_files_names = VOCAB_FILES_NAMES
diff --git a/src/transformers/models/electra/tokenization_electra_fast.py b/src/transformers/models/electra/tokenization_electra_fast.py
index 41c7bd5536..48a28cc98b 100644
--- a/src/transformers/models/electra/tokenization_electra_fast.py
+++ b/src/transformers/models/electra/tokenization_electra_fast.py
@@ -62,11 +62,10 @@ class ElectraTokenizerFast(BertTokenizerFast):
r"""
Construct a "fast" ELECTRA tokenizer (backed by HuggingFace's *tokenizers* library).
- [`ElectraTokenizerFast`] is identical to [`BertTokenizerFast`] and runs
- end-to-end tokenization: punctuation splitting and wordpiece.
+ [`ElectraTokenizerFast`] is identical to [`BertTokenizerFast`] and runs end-to-end tokenization: punctuation
+ splitting and wordpiece.
- Refer to superclass [`BertTokenizerFast`] for usage examples and documentation concerning
- parameters.
+ Refer to superclass [`BertTokenizerFast`] for usage examples and documentation concerning parameters.
"""
vocab_files_names = VOCAB_FILES_NAMES
pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
diff --git a/src/transformers/models/encoder_decoder/configuration_encoder_decoder.py b/src/transformers/models/encoder_decoder/configuration_encoder_decoder.py
index 4fc7f6b563..833ff5d759 100644
--- a/src/transformers/models/encoder_decoder/configuration_encoder_decoder.py
+++ b/src/transformers/models/encoder_decoder/configuration_encoder_decoder.py
@@ -25,21 +25,21 @@ logger = logging.get_logger(__name__)
class EncoderDecoderConfig(PretrainedConfig):
r"""
- [`EncoderDecoderConfig`] is the configuration class to store the configuration of a
- [`EncoderDecoderModel`]. It is used to instantiate an Encoder Decoder model according to the
- specified arguments, defining the encoder and decoder configs.
+ [`EncoderDecoderConfig`] is the configuration class to store the configuration of a [`EncoderDecoderModel`]. It is
+ used to instantiate an Encoder Decoder model according to the specified arguments, defining the encoder and decoder
+ configs.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
kwargs (*optional*):
Dictionary of keyword arguments. Notably:
- - **encoder** ([`PretrainedConfig`], *optional*) -- An instance of a configuration
- object that defines the encoder config.
- - **decoder** ([`PretrainedConfig`], *optional*) -- An instance of a configuration
- object that defines the decoder config.
+ - **encoder** ([`PretrainedConfig`], *optional*) -- An instance of a configuration object that defines
+ the encoder config.
+ - **decoder** ([`PretrainedConfig`], *optional*) -- An instance of a configuration object that defines
+ the decoder config.
Examples:
@@ -93,8 +93,8 @@ class EncoderDecoderConfig(PretrainedConfig):
cls, encoder_config: PretrainedConfig, decoder_config: PretrainedConfig, **kwargs
) -> PretrainedConfig:
r"""
- Instantiate a [`EncoderDecoderConfig`] (or a derived class) from a pre-trained encoder model
- configuration and decoder model configuration.
+ Instantiate a [`EncoderDecoderConfig`] (or a derived class) from a pre-trained encoder model configuration and
+ decoder model configuration.
Returns:
[`EncoderDecoderConfig`]: An instance of a configuration object
diff --git a/src/transformers/models/encoder_decoder/modeling_encoder_decoder.py b/src/transformers/models/encoder_decoder/modeling_encoder_decoder.py
index 96205ea334..44c2568719 100644
--- a/src/transformers/models/encoder_decoder/modeling_encoder_decoder.py
+++ b/src/transformers/models/encoder_decoder/modeling_encoder_decoder.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" Classes to support Encoder-Decoder architectures """
+""" Classes to support Encoder-Decoder architectures"""
import warnings
from typing import Optional
@@ -45,30 +45,30 @@ DEPRECATION_WARNING = (
ENCODER_DECODER_START_DOCSTRING = r"""
This class can be used to initialize a sequence-to-sequence model with any pretrained autoencoding model as the
encoder and any pretrained autoregressive model as the decoder. The encoder is loaded via
- [`~AutoModel.from_pretrained`] function and the decoder is loaded via
- [`~AutoModelForCausalLM.from_pretrained`] function. Cross-attention layers are automatically added
- to the decoder and should be fine-tuned on a downstream generative task, like summarization.
+ [`~AutoModel.from_pretrained`] function and the decoder is loaded via [`~AutoModelForCausalLM.from_pretrained`]
+ function. Cross-attention layers are automatically added to the decoder and should be fine-tuned on a downstream
+ generative task, like summarization.
The effectiveness of initializing sequence-to-sequence models with pretrained checkpoints for sequence generation
- tasks was shown in [Leveraging Pre-trained Checkpoints for Sequence Generation Tasks](https://arxiv.org/abs/1907.12461) by Sascha Rothe, Shashi Narayan, Aliaksei Severyn. Michael Matena, Yanqi
+ tasks was shown in [Leveraging Pre-trained Checkpoints for Sequence Generation
+ Tasks](https://arxiv.org/abs/1907.12461) by Sascha Rothe, Shashi Narayan, Aliaksei Severyn. Michael Matena, Yanqi
Zhou, Wei Li, Peter J. Liu.
After such an Encoder Decoder model has been trained/fine-tuned, it can be saved/loaded just like any other models
(see the examples for more information).
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
Parameters:
config ([`EncoderDecoderConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
ENCODER_DECODER_INPUTS_DOCSTRING = r"""
@@ -76,9 +76,8 @@ ENCODER_DECODER_INPUTS_DOCSTRING = r"""
input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`PreTrainedTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`PreTrainedTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -91,45 +90,45 @@ ENCODER_DECODER_INPUTS_DOCSTRING = r"""
decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
Indices of decoder input sequence tokens in the vocabulary.
- Indices can be obtained using [`PreTrainedTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`PreTrainedTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
`past_key_values`).
- For training, `decoder_input_ids` are automatically created by the model by shifting the `labels`
- to the right, replacing -100 by the `pad_token_id` and prepending them with the
- `decoder_start_token_id`.
+ For training, `decoder_input_ids` are automatically created by the model by shifting the `labels` to the
+ right, replacing -100 by the `pad_token_id` and prepending them with the `decoder_start_token_id`.
decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
- Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will
- also be used by default.
+ Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
+ be used by default.
encoder_outputs (`tuple(torch.FloatTensor)`, *optional*):
- This tuple must consist of (`last_hidden_state`, *optional*: `hidden_states`, *optional*:
- `attentions`) `last_hidden_state` (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`) is a tensor of hidden-states at the output of the last layer of the
- encoder. Used in the cross-attention of the decoder.
+ This tuple must consist of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`)
+ `last_hidden_state` (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`) is a tensor
+ of hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the
+ decoder.
past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ `decoder_input_ids` of shape `(batch_size, sequence_length)`.
inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*):
Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded
- representation. This is useful if you want more control over how to convert `decoder_input_ids`
- indices into associated vectors than the model's internal embedding lookup matrix.
+ representation. This is useful if you want more control over how to convert `decoder_input_ids` indices
+ into associated vectors than the model's internal embedding lookup matrix.
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss for the decoder. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
+ Labels for computing the masked language modeling loss for the decoder. Indices should be in `[-100, 0,
+ ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
(masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
use_cache (`bool`, *optional*):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`).
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`).
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -137,8 +136,7 @@ ENCODER_DECODER_INPUTS_DOCSTRING = r"""
Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
more detail.
return_dict (`bool`, *optional*):
- If set to `True`, the model will return a [`~file_utils.Seq2SeqLMOutput`] instead of a
- plain tuple.
+ If set to `True`, the model will return a [`~file_utils.Seq2SeqLMOutput`] instead of a plain tuple.
kwargs: (*optional*) Remaining dictionary of keyword arguments. Keyword arguments come in two flavors:
- Without a prefix which will be input as `**encoder_kwargs` for the encoder forward function.
@@ -167,9 +165,9 @@ def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int, decoder_start
@add_start_docstrings(ENCODER_DECODER_START_DOCSTRING)
class EncoderDecoderModel(PreTrainedModel):
r"""
- [`EncoderDecoderModel`] is a generic model class that will be instantiated as a transformer
- architecture with one of the base model classes of the library as encoder and another one as decoder when created
- with the :meth*~transformers.AutoModel.from_pretrained* class method for the encoder and
+ [`EncoderDecoderModel`] is a generic model class that will be instantiated as a transformer architecture with one
+ of the base model classes of the library as encoder and another one as decoder when created with the
+ :meth*~transformers.AutoModel.from_pretrained* class method for the encoder and
:meth*~transformers.AutoModelForCausalLM.from_pretrained* class method for the decoder.
"""
config_class = EncoderDecoderConfig
@@ -291,35 +289,35 @@ class EncoderDecoderModel(PreTrainedModel):
checkpoints.
- The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated). To
- train the model, you need to first set it back in training mode with `model.train()`.
+ The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated). To train
+ the model, you need to first set it back in training mode with `model.train()`.
Params:
encoder_pretrained_model_name_or_path (:obj: *str*, *optional*):
Information necessary to initiate the encoder. Can be either:
- A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co.
- Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under
- a user or organization name, like `dbmdz/bert-base-german-cased`.
+ Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under a
+ user or organization name, like `dbmdz/bert-base-german-cased`.
- A path to a *directory* containing model weights saved using
[`~PreTrainedModel.save_pretrained`], e.g., `./my_model_directory/`.
- A path or url to a *tensorflow index checkpoint file* (e.g, `./tf_model/model.ckpt.index`). In
- this case, `from_tf` should be set to `True` and a configuration object should be provided
- as `config` argument. This loading path is slower than converting the TensorFlow checkpoint in
- a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards.
+ this case, `from_tf` should be set to `True` and a configuration object should be provided as
+ `config` argument. This loading path is slower than converting the TensorFlow checkpoint in a
+ PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards.
decoder_pretrained_model_name_or_path (:obj: *str*, *optional*, defaults to *None*):
Information necessary to initiate the decoder. Can be either:
- A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co.
- Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under
- a user or organization name, like `dbmdz/bert-base-german-cased`.
+ Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under a
+ user or organization name, like `dbmdz/bert-base-german-cased`.
- A path to a *directory* containing model weights saved using
[`~PreTrainedModel.save_pretrained`], e.g., `./my_model_directory/`.
- A path or url to a *tensorflow index checkpoint file* (e.g, `./tf_model/model.ckpt.index`). In
- this case, `from_tf` should be set to `True` and a configuration object should be provided
- as `config` argument. This loading path is slower than converting the TensorFlow checkpoint in
- a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards.
+ this case, `from_tf` should be set to `True` and a configuration object should be provided as
+ `config` argument. This loading path is slower than converting the TensorFlow checkpoint in a
+ PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards.
model_args (remaining positional arguments, *optional*):
All remaining positional arguments will be passed to the underlying model's `__init__` method.
diff --git a/src/transformers/models/encoder_decoder/modeling_flax_encoder_decoder.py b/src/transformers/models/encoder_decoder/modeling_flax_encoder_decoder.py
index 186b2ee527..1a1994392f 100644
--- a/src/transformers/models/encoder_decoder/modeling_flax_encoder_decoder.py
+++ b/src/transformers/models/encoder_decoder/modeling_flax_encoder_decoder.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" Classes to support Flax Encoder-Decoder architectures """
+""" Classes to support Flax Encoder-Decoder architectures"""
import os
@@ -41,32 +41,33 @@ _CONFIG_FOR_DOC = "EncoderDecoderConfig"
ENCODER_DECODER_START_DOCSTRING = r"""
This class can be used to initialize a sequence-to-sequence model with any pretrained autoencoding model as the
encoder and any pretrained autoregressive model as the decoder. The encoder is loaded via
- [`~AutoModel.from_pretrained`] function and the decoder is loaded via
- [`~AutoModelForCausalLM.from_pretrained`] function. Cross-attention layers are automatically added
- to the decoder and should be fine-tuned on a downstream generative task, like summarization.
+ [`~AutoModel.from_pretrained`] function and the decoder is loaded via [`~AutoModelForCausalLM.from_pretrained`]
+ function. Cross-attention layers are automatically added to the decoder and should be fine-tuned on a downstream
+ generative task, like summarization.
The effectiveness of initializing sequence-to-sequence models with pretrained checkpoints for sequence generation
- tasks was shown in [Leveraging Pre-trained Checkpoints for Sequence Generation Tasks](https://arxiv.org/abs/1907.12461) by Sascha Rothe, Shashi Narayan, Aliaksei Severyn. Michael Matena, Yanqi
+ tasks was shown in [Leveraging Pre-trained Checkpoints for Sequence Generation
+ Tasks](https://arxiv.org/abs/1907.12461) by Sascha Rothe, Shashi Narayan, Aliaksei Severyn. Michael Matena, Yanqi
Zhou, Wei Li, Peter J. Liu.
After such an Encoder Decoder model has been trained/fine-tuned, it can be saved/loaded just like any other models
(see the examples for more information).
- This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading or saving, resizing the input
- embeddings, pruning heads etc.)
+ This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a Flax Linen [flax.nn.Module](https://flax.readthedocs.io/en/latest/_autosummary/flax.nn.module.html) subclass. Use it as a regular Flax
- Module and refer to the Flax documentation for all matter related to general usage and behavior.
+ This model is also a Flax Linen
+ [flax.nn.Module](https://flax.readthedocs.io/en/latest/_autosummary/flax.nn.module.html) subclass. Use it as a
+ regular Flax Module and refer to the Flax documentation for all matter related to general usage and behavior.
Parameters:
config ([`EncoderDecoderConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the
- model weights.
+ configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the model weights.
dtype (`jax.numpy.dtype`, *optional*, defaults to `jax.numpy.float32`):
- The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on
- GPUs) and `jax.numpy.bfloat16` (on TPUs).
+ The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on GPUs) and
+ `jax.numpy.bfloat16` (on TPUs).
This can be used to enable mixed-precision training or half-precision inference on GPUs or TPUs. If
specified all the computation will be performed with the given `dtype`.
@@ -74,8 +75,8 @@ ENCODER_DECODER_START_DOCSTRING = r"""
**Note that this only specifies the dtype of the computation and does not influence the dtype of model
parameters.**
- If you wish to change the dtype of the model parameters, see
- [`~FlaxPreTrainedModel.to_fp16`] and [`~FlaxPreTrainedModel.to_bf16`].
+ If you wish to change the dtype of the model parameters, see [`~FlaxPreTrainedModel.to_fp16`] and
+ [`~FlaxPreTrainedModel.to_bf16`].
"""
ENCODER_DECODER_INPUTS_DOCSTRING = r"""
@@ -84,9 +85,8 @@ ENCODER_DECODER_INPUTS_DOCSTRING = r"""
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
it.
- Indices can be obtained using [`PreTrainedTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`PreTrainedTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`jnp.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
@@ -99,20 +99,20 @@ ENCODER_DECODER_INPUTS_DOCSTRING = r"""
decoder_input_ids (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`, *optional*):
Indices of decoder input sequence tokens in the vocabulary.
- Indices can be obtained using [`PreTrainedTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`PreTrainedTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are decoder input IDs?](../glossary#decoder-input-ids)
- For sequence to sequence training, `decoder_input_ids` should be provided. If no
- `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to
- the right for denoising pre-training.
+ For sequence to sequence training, `decoder_input_ids` should be provided. If no `decoder_input_ids` is
+ provided, the model will create this tensor by shifting the `input_ids` to the right for denoising
+ pre-training.
decoder_attention_mask (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`, *optional*):
- Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will
- also be used by default.
+ Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
+ be used by default.
position_ids (`numpy.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.encoder.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.encoder.max_position_embeddings - 1]`.
decoder_position_ids (`numpy.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the
range `[0, config.decoder.max_position_embeddings - 1]`.
@@ -123,8 +123,7 @@ ENCODER_DECODER_INPUTS_DOCSTRING = r"""
Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
more detail.
return_dict (`bool`, *optional*):
- If set to `True`, the model will return a [`~file_utils.FlaxSeq2SeqLMOutput`] instead
- of a plain tuple.
+ If set to `True`, the model will return a [`~file_utils.FlaxSeq2SeqLMOutput`] instead of a plain tuple.
"""
ENCODER_DECODER_ENCODE_INPUTS_DOCSTRING = r"""
@@ -133,9 +132,8 @@ ENCODER_DECODER_ENCODE_INPUTS_DOCSTRING = r"""
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
it.
- Indices can be obtained using [`PreTrainedTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`PreTrainedTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`jnp.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
@@ -146,7 +144,8 @@ ENCODER_DECODER_ENCODE_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
position_ids (`numpy.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.encoder.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.encoder.max_position_embeddings - 1]`.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -154,8 +153,7 @@ ENCODER_DECODER_ENCODE_INPUTS_DOCSTRING = r"""
Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
more detail.
return_dict (`bool`, *optional*):
- If set to `True`, the model will return a [`~file_utils.FlaxBaseModelOutput`] instead
- of a plain tuple.
+ If set to `True`, the model will return a [`~file_utils.FlaxBaseModelOutput`] instead of a plain tuple.
"""
ENCODER_DECODER_DECODE_INPUTS_DOCSTRING = r"""
@@ -163,23 +161,21 @@ ENCODER_DECODER_DECODE_INPUTS_DOCSTRING = r"""
decoder_input_ids (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`, *optional*):
Indices of decoder input sequence tokens in the vocabulary.
- Indices can be obtained using [`PreTrainedTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`PreTrainedTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are decoder input IDs?](../glossary#decoder-input-ids)
If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
`past_key_values`).
- For sequence to sequence training, `decoder_input_ids` should be provided. If no
- `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to
- the right for denoising pre-training.
+ For sequence to sequence training, `decoder_input_ids` should be provided. If no `decoder_input_ids` is
+ provided, the model will create this tensor by shifting the `input_ids` to the right for denoising
+ pre-training.
encoder_outputs (`tuple(tuple(jnp.ndarray)`):
- Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*:
- `attentions`) `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`,
- *optional*) is a sequence of hidden-states at the output of the last layer of the encoder. Used in the
- cross-attention of the decoder.
+ Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`)
+ `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of
+ hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder.
encoder_attention_mask (`jnp.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
@@ -188,8 +184,8 @@ ENCODER_DECODER_DECODE_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
decoder_attention_mask (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`, *optional*):
- Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will
- also be used by default.
+ Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
+ be used by default.
decoder_position_ids (`numpy.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the
range `[0, config.decoder.max_position_embeddings - 1]`.
@@ -203,8 +199,8 @@ ENCODER_DECODER_DECODE_INPUTS_DOCSTRING = r"""
Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
more detail.
return_dict (`bool`, *optional*):
- If set to `True`, the model will return a
- [`~file_utils.FlaxCausalLMOutputWithCrossAttentions`] instead of a plain tuple.
+ If set to `True`, the model will return a [`~file_utils.FlaxCausalLMOutputWithCrossAttentions`] instead of
+ a plain tuple.
"""
@@ -305,10 +301,10 @@ class FlaxEncoderDecoderModule(nn.Module):
@add_start_docstrings(ENCODER_DECODER_START_DOCSTRING)
class FlaxEncoderDecoderModel(FlaxPreTrainedModel):
r"""
- [`FlaxEncoderDecoderModel`] is a generic model class that will be instantiated as a transformer
- architecture with the module (flax.nn.Module) of one of the base model classes of the library as encoder module and
- another one as decoder module when created with the :meth*~transformers.FlaxAutoModel.from_pretrained* class method
- for the encoder and :meth*~transformers.FlaxAutoModelForCausalLM.from_pretrained* class method for the decoder.
+ [`FlaxEncoderDecoderModel`] is a generic model class that will be instantiated as a transformer architecture with
+ the module (flax.nn.Module) of one of the base model classes of the library as encoder module and another one as
+ decoder module when created with the :meth*~transformers.FlaxAutoModel.from_pretrained* class method for the
+ encoder and :meth*~transformers.FlaxAutoModelForCausalLM.from_pretrained* class method for the decoder.
"""
config_class = EncoderDecoderConfig
base_model_prefix = "encoder_decoder"
@@ -380,9 +376,10 @@ class FlaxEncoderDecoderModel(FlaxPreTrainedModel):
maximum possible length for auto-regressive decoding. Defines the sequence length of the initialized
cache.
encoder_outputs (`Union[FlaxBaseModelOutput, tuple(tuple(jnp.ndarray)]`):
- `encoder_outputs` consists of (`last_hidden_state`, *optional*: `hidden_states`,
- *optional*: `attentions`). `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of hidden-states at the output of the last layer of the
- encoder. Used in the cross-attention of the decoder.
+ `encoder_outputs` consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*:
+ `attentions`). `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*)
+ is a sequence of hidden-states at the output of the last layer of the encoder. Used in the
+ cross-attention of the decoder.
"""
# init input variables to retrieve cache
decoder_input_ids = jnp.ones((batch_size, max_length), dtype="i4")
@@ -753,8 +750,8 @@ class FlaxEncoderDecoderModel(FlaxPreTrainedModel):
Information necessary to initiate the encoder. Can be either:
- A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co.
- Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under
- a user or organization name, like `dbmdz/bert-base-german-cased`.
+ Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under a
+ user or organization name, like `dbmdz/bert-base-german-cased`.
- A path to a *directory* containing model weights saved using
[`~FlaxPreTrainedModel.save_pretrained`], e.g., `./my_model_directory/`.
@@ -762,8 +759,8 @@ class FlaxEncoderDecoderModel(FlaxPreTrainedModel):
Information necessary to initiate the decoder. Can be either:
- A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co.
- Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under
- a user or organization name, like `dbmdz/bert-base-german-cased`.
+ Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under a
+ user or organization name, like `dbmdz/bert-base-german-cased`.
- A path to a *directory* containing model weights saved using
[`~FlaxPreTrainedModel.save_pretrained`], e.g., `./my_model_directory/`.
diff --git a/src/transformers/models/encoder_decoder/modeling_tf_encoder_decoder.py b/src/transformers/models/encoder_decoder/modeling_tf_encoder_decoder.py
index 4735d94a3f..e6c88cc02f 100644
--- a/src/transformers/models/encoder_decoder/modeling_tf_encoder_decoder.py
+++ b/src/transformers/models/encoder_decoder/modeling_tf_encoder_decoder.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" Classes to support TF Encoder-Decoder architectures """
+""" Classes to support TF Encoder-Decoder architectures"""
import tempfile
@@ -43,30 +43,30 @@ _CONFIG_FOR_DOC = "EncoderDecoderConfig"
ENCODER_DECODER_START_DOCSTRING = r"""
This class can be used to initialize a sequence-to-sequence model with any pretrained autoencoding model as the
encoder and any pretrained autoregressive model as the decoder. The encoder is loaded via
- [`~TFAutoModel.from_pretrained`] function and the decoder is loaded via
- [`~TFAutoModelForCausalLM.from_pretrained`] function. Cross-attention layers are automatically
- added to the decoder and should be fine-tuned on a downstream generative task, like summarization.
+ [`~TFAutoModel.from_pretrained`] function and the decoder is loaded via [`~TFAutoModelForCausalLM.from_pretrained`]
+ function. Cross-attention layers are automatically added to the decoder and should be fine-tuned on a downstream
+ generative task, like summarization.
The effectiveness of initializing sequence-to-sequence models with pretrained checkpoints for sequence generation
- tasks was shown in [Leveraging Pre-trained Checkpoints for Sequence Generation Tasks](https://arxiv.org/abs/1907.12461) by Sascha Rothe, Shashi Narayan, Aliaksei Severyn. Michael Matena, Yanqi
+ tasks was shown in [Leveraging Pre-trained Checkpoints for Sequence Generation
+ Tasks](https://arxiv.org/abs/1907.12461) by Sascha Rothe, Shashi Narayan, Aliaksei Severyn. Michael Matena, Yanqi
Zhou, Wei Li, Peter J. Liu.
After such an Encoder Decoder model has been trained/fine-tuned, it can be saved/loaded just like any other models
(see the examples for more information).
- This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading or saving, resizing the input
- embeddings, pruning heads etc.)
+ This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use
- it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage
- and behavior.
+ This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it
+ as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and
+ behavior.
Parameters:
config ([`EncoderDecoderConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~TFPreTrainedModel.from_pretrained`] method to load the
- model weights.
+ configuration. Check out the [`~TFPreTrainedModel.from_pretrained`] method to load the model weights.
"""
ENCODER_DECODER_INPUTS_DOCSTRING = r"""
@@ -74,9 +74,8 @@ ENCODER_DECODER_INPUTS_DOCSTRING = r"""
input_ids (`np.ndarray`, `tf.Tensor`, `List[tf.Tensor]` ``Dict[str, tf.Tensor]` or `Dict[str, np.ndarray]` and each example must have the shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`PreTrainedTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`PreTrainedTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`np.ndarray` or `tf.Tensor` of shape `({0})`, *optional*):
@@ -89,9 +88,8 @@ ENCODER_DECODER_INPUTS_DOCSTRING = r"""
decoder_input_ids (`np.ndarray` or `tf.Tensor` of shape `(batch_size, target_sequence_length)`, *optional*):
Indices of decoder input sequence tokens in the vocabulary.
- Indices can be obtained using [`PreTrainedTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`PreTrainedTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
@@ -99,36 +97,36 @@ ENCODER_DECODER_INPUTS_DOCSTRING = r"""
`past_key_values`).
Provide for sequence to sequence training to the decoder. Indices can be obtained using
- [`PreTrainedTokenizer`]. See [`PreTrainedTokenizer.encode`] and
- [`PreTrainedTokenizer.__call__`] for details.
+ [`PreTrainedTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
+ details.
decoder_attention_mask (`np.ndarray` or `tf.Tensor` of shape `(batch_size, target_sequence_length)`, *optional*):
- Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will
- also be used by default.
+ Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
+ be used by default.
encoder_outputs (`tuple(tuple(tf.Tensor)`, *optional*):
- This tuple must consist of (`last_hidden_state`, *optional*: `hidden_states`, *optional*:
- `attentions`) `last_hidden_state` (`tf.Tensor` of shape `({0}, hidden_size)`) is a
- tensor of hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the
- decoder.
+ This tuple must consist of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`)
+ `last_hidden_state` (`tf.Tensor` of shape `({0}, hidden_size)`) is a tensor of hidden-states at the output
+ of the last layer of the encoder. Used in the cross-attention of the decoder.
past_key_values (`tuple(tuple(tf.Tensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all `decoder_input_ids` of shape `({0})`.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ `decoder_input_ids` of shape `({0})`.
inputs_embeds (`np.ndarray` or `tf.Tensor` of shape `({0}, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
decoder_inputs_embeds (`np.ndarray` or `tf.Tensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*):
Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded
- representation. This is useful if you want more control over how to convert `decoder_input_ids`
- indices into associated vectors than the model's internal embedding lookup matrix.
+ representation. This is useful if you want more control over how to convert `decoder_input_ids` indices
+ into associated vectors than the model's internal embedding lookup matrix.
labels (`np.ndarray` or `tf.Tensor` of shape `({0})`, *optional*):
- Labels for computing the masked language modeling loss for the decoder. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
+ Labels for computing the masked language modeling loss for the decoder. Indices should be in `[-100, 0,
+ ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
(masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
use_cache (`bool`, *optional*):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`).
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`).
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -136,8 +134,7 @@ ENCODER_DECODER_INPUTS_DOCSTRING = r"""
Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
more detail.
return_dict (`bool`, *optional*):
- If set to `True`, the model will return a [`~file_utils.Seq2SeqLMOutput`] instead of a
- plain tuple.
+ If set to `True`, the model will return a [`~file_utils.Seq2SeqLMOutput`] instead of a plain tuple.
training (`bool`, *optional*, defaults to `False`):
Whether or not to use the model in training mode (some modules like dropout modules have different
behaviors between training and evaluation).
@@ -151,9 +148,9 @@ ENCODER_DECODER_INPUTS_DOCSTRING = r"""
@add_start_docstrings(ENCODER_DECODER_START_DOCSTRING)
class TFEncoderDecoderModel(TFPreTrainedModel):
r"""
- [`TFEncoderDecoder`] is a generic model class that will be instantiated as a transformer
- architecture with one of the base model classes of the library as encoder and another one as decoder when created
- with the :meth*~transformers.TFAutoModel.from_pretrained* class method for the encoder and
+ [`TFEncoderDecoder`] is a generic model class that will be instantiated as a transformer architecture with one of
+ the base model classes of the library as encoder and another one as decoder when created with the
+ :meth*~transformers.TFAutoModel.from_pretrained* class method for the encoder and
:meth*~transformers.TFAutoModelForCausalLM.from_pretrained* class method for the decoder.
"""
config_class = EncoderDecoderConfig
@@ -315,8 +312,8 @@ class TFEncoderDecoderModel(TFPreTrainedModel):
Information necessary to initiate the encoder. Can be either:
- A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co.
- Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under
- a user or organization name, like `dbmdz/bert-base-german-cased`.
+ Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under a
+ user or organization name, like `dbmdz/bert-base-german-cased`.
- A path to a *directory* containing model weights saved using
[`~TFPreTrainedModel.save_pretrained`], e.g., `./my_model_directory/`.
- A path or url to a *pytorch index checkpoint file* (e.g, `./pt_model/`). In this case,
@@ -326,8 +323,8 @@ class TFEncoderDecoderModel(TFPreTrainedModel):
Information necessary to initiate the decoder. Can be either:
- A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co.
- Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under
- a user or organization name, like `dbmdz/bert-base-german-cased`.
+ Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under a
+ user or organization name, like `dbmdz/bert-base-german-cased`.
- A path to a *directory* containing model weights saved using
[`~TFPreTrainedModel.save_pretrained`], e.g., `./my_model_directory/`.
- A path or url to a *pytorch checkpoint file* (e.g, `./pt_model/`). In this case,
diff --git a/src/transformers/models/flaubert/configuration_flaubert.py b/src/transformers/models/flaubert/configuration_flaubert.py
index 14509347f4..037e860069 100644
--- a/src/transformers/models/flaubert/configuration_flaubert.py
+++ b/src/transformers/models/flaubert/configuration_flaubert.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" Flaubert configuration, based on XLM. """
+""" Flaubert configuration, based on XLM."""
from ...utils import logging
from ..xlm.configuration_xlm import XLMConfig
@@ -30,12 +30,11 @@ FLAUBERT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class FlaubertConfig(XLMConfig):
"""
- This is the configuration class to store the configuration of a [`FlaubertModel`] or a
- [`TFFlaubertModel`]. It is used to instantiate a FlauBERT model according to the specified
- arguments, defining the model architecture.
+ This is the configuration class to store the configuration of a [`FlaubertModel`] or a [`TFFlaubertModel`]. It is
+ used to instantiate a FlauBERT model according to the specified arguments, defining the model architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
pre_norm (`bool`, *optional*, defaults to `False`):
@@ -46,8 +45,7 @@ class FlaubertConfig(XLMConfig):
Structured Dropout. ICLR 2020)
vocab_size (`int`, *optional*, defaults to 30145):
Vocabulary size of the FlauBERT model. Defines the number of different tokens that can be represented by
- the `inputs_ids` passed when calling [`FlaubertModel`] or
- [`TFFlaubertModel`].
+ the `inputs_ids` passed when calling [`FlaubertModel`] or [`TFFlaubertModel`].
emb_dim (`int`, *optional*, defaults to 2048):
Dimensionality of the encoder layers and the pooler layer.
n_layer (`int`, *optional*, defaults to 12):
@@ -72,8 +70,8 @@ class FlaubertConfig(XLMConfig):
The number of languages the model handles. Set to 1 for monolingual models.
use_lang_emb (`bool`, *optional*, defaults to `True`)
Whether to use language embeddings. Some models use additional language embeddings, see [the multilingual
- models page](http://huggingface.co/transformers/multilingual.html#xlm-language-embeddings) for
- information on how to use them.
+ models page](http://huggingface.co/transformers/multilingual.html#xlm-language-embeddings) for information
+ on how to use them.
max_position_embeddings (`int`, *optional*, defaults to 512):
The maximum sequence length that this model might ever be used with. Typically set this to something large
just in case (e.g., 512 or 1024 or 2048).
diff --git a/src/transformers/models/flaubert/modeling_flaubert.py b/src/transformers/models/flaubert/modeling_flaubert.py
index d13ed14bb8..edbf25ec43 100644
--- a/src/transformers/models/flaubert/modeling_flaubert.py
+++ b/src/transformers/models/flaubert/modeling_flaubert.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" PyTorch Flaubert model, based on XLM. """
+""" PyTorch Flaubert model, based on XLM."""
import random
@@ -54,19 +54,18 @@ FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST = [
FLAUBERT_START_DOCSTRING = r"""
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
Parameters:
config ([`FlaubertConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
FLAUBERT_INPUTS_DOCSTRING = r"""
@@ -74,9 +73,8 @@ FLAUBERT_INPUTS_DOCSTRING = r"""
input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`FlaubertTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`FlaubertTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -87,25 +85,27 @@ FLAUBERT_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
lengths (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Length of each sentence that can be used to avoid performing attention on padding token indices. You can
- also use `attention_mask` for the same result (see above), kept here for compatibility. Indices
- selected in `[0, ..., input_ids.size(-1)]`:
+ also use `attention_mask` for the same result (see above), kept here for compatibility. Indices selected in
+ `[0, ..., input_ids.size(-1)]`:
cache (`Dict[str, torch.FloatTensor]`, *optional*):
Dictionary strings to `torch.FloatTensor` that contains precomputed hidden-states (key and values in the
- attention blocks) as computed by the model (see `cache` output below). Can be used to speed up
- sequential decoding. The dictionary object will be modified in-place during the forward pass to add newly
- computed hidden-states.
+ attention blocks) as computed by the model (see `cache` output below). Can be used to speed up sequential
+ decoding. The dictionary object will be modified in-place during the forward pass to add newly computed
+ hidden-states.
head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:
@@ -113,9 +113,9 @@ FLAUBERT_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -325,8 +325,8 @@ class FlaubertModel(XLMModel):
)
class FlaubertWithLMHeadModel(XLMWithLMHeadModel):
"""
- This class overrides [`XLMWithLMHeadModel`]. Please check the superclass for the appropriate
- documentation alongside usage examples.
+ This class overrides [`XLMWithLMHeadModel`]. Please check the superclass for the appropriate documentation
+ alongside usage examples.
"""
config_class = FlaubertConfig
@@ -347,8 +347,8 @@ class FlaubertWithLMHeadModel(XLMWithLMHeadModel):
)
class FlaubertForSequenceClassification(XLMForSequenceClassification):
"""
- This class overrides [`XLMForSequenceClassification`]. Please check the superclass for the
- appropriate documentation alongside usage examples.
+ This class overrides [`XLMForSequenceClassification`]. Please check the superclass for the appropriate
+ documentation alongside usage examples.
"""
config_class = FlaubertConfig
@@ -369,8 +369,8 @@ class FlaubertForSequenceClassification(XLMForSequenceClassification):
)
class FlaubertForTokenClassification(XLMForTokenClassification):
"""
- This class overrides [`XLMForTokenClassification`]. Please check the superclass for the
- appropriate documentation alongside usage examples.
+ This class overrides [`XLMForTokenClassification`]. Please check the superclass for the appropriate documentation
+ alongside usage examples.
"""
config_class = FlaubertConfig
@@ -391,8 +391,8 @@ class FlaubertForTokenClassification(XLMForTokenClassification):
)
class FlaubertForQuestionAnsweringSimple(XLMForQuestionAnsweringSimple):
"""
- This class overrides [`XLMForQuestionAnsweringSimple`]. Please check the superclass for the
- appropriate documentation alongside usage examples.
+ This class overrides [`XLMForQuestionAnsweringSimple`]. Please check the superclass for the appropriate
+ documentation alongside usage examples.
"""
config_class = FlaubertConfig
@@ -413,8 +413,8 @@ class FlaubertForQuestionAnsweringSimple(XLMForQuestionAnsweringSimple):
)
class FlaubertForQuestionAnswering(XLMForQuestionAnswering):
"""
- This class overrides [`XLMForQuestionAnswering`]. Please check the superclass for the
- appropriate documentation alongside usage examples.
+ This class overrides [`XLMForQuestionAnswering`]. Please check the superclass for the appropriate documentation
+ alongside usage examples.
"""
config_class = FlaubertConfig
@@ -435,8 +435,8 @@ class FlaubertForQuestionAnswering(XLMForQuestionAnswering):
)
class FlaubertForMultipleChoice(XLMForMultipleChoice):
"""
- This class overrides [`XLMForMultipleChoice`]. Please check the superclass for the appropriate
- documentation alongside usage examples.
+ This class overrides [`XLMForMultipleChoice`]. Please check the superclass for the appropriate documentation
+ alongside usage examples.
"""
config_class = FlaubertConfig
diff --git a/src/transformers/models/flaubert/modeling_tf_flaubert.py b/src/transformers/models/flaubert/modeling_tf_flaubert.py
index 97089fc666..87c1c7e6b0 100644
--- a/src/transformers/models/flaubert/modeling_tf_flaubert.py
+++ b/src/transformers/models/flaubert/modeling_tf_flaubert.py
@@ -62,13 +62,13 @@ TF_FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST = [
FLAUBERT_START_DOCSTRING = r"""
- This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading or saving, resizing the input
- embeddings, pruning heads etc.)
+ This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use
- it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage
- and behavior.
+ This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it
+ as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and
+ behavior.
@@ -77,11 +77,11 @@ FLAUBERT_START_DOCSTRING = r"""
- having all inputs as keyword arguments (like PyTorch models), or
- having all inputs as a list, tuple or dict in the first positional arguments.
- This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all
- the tensors in the first argument of the model call function: `model(inputs)`.
+ This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all the
+ tensors in the first argument of the model call function: `model(inputs)`.
- If you choose this second option, there are three possibilities you can use to gather all the input Tensors in
- the first positional argument :
+ If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the
+ first positional argument :
- a single Tensor with `input_ids` only and nothing else: `model(inputs_ids)`
- a list of varying length with one or several input Tensors IN THE ORDER given in the docstring:
@@ -94,8 +94,7 @@ FLAUBERT_START_DOCSTRING = r"""
Parameters:
config ([`FlaubertConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
FLAUBERT_INPUTS_DOCSTRING = r"""
@@ -103,9 +102,8 @@ FLAUBERT_INPUTS_DOCSTRING = r"""
input_ids (`Numpy array` or `tf.Tensor` of shape `(batch_size, sequence_length)`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`FlaubertTokenizer`]. See
- [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for
- details.
+ Indices can be obtained using [`FlaubertTokenizer`]. See [`PreTrainedTokenizer.__call__`] and
+ [`PreTrainedTokenizer.encode`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`Numpy array` or `tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -124,14 +122,16 @@ FLAUBERT_INPUTS_DOCSTRING = r"""
See usage examples detailed in the [multilingual documentation](../multilingual).
token_type_ids (`tf.Tensor` or `Numpy array` of shape `(batch_size, sequence_length)`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- `0` corresponds to a *sentence A* token,
- `1` corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`tf.Tensor` or `Numpy array` of shape `(batch_size, sequence_length)`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
lengths (`tf.Tensor` or `Numpy array` of shape `(batch_size,)`, *optional*):
@@ -140,8 +140,8 @@ FLAUBERT_INPUTS_DOCSTRING = r"""
`[0, ..., input_ids.size(-1)]`:
cache (`Dict[str, tf.Tensor]`, *optional*):
Dictionary string to `tf.FloatTensor` that contains precomputed hidden states (key and values in the
- attention blocks) as computed by the model (see `cache` output below). Can be used to speed up
- sequential decoding.
+ attention blocks) as computed by the model (see `cache` output below). Can be used to speed up sequential
+ decoding.
The dictionary object will be modified in-place during the forward pass to add newly computed
hidden-states.
@@ -152,9 +152,9 @@ FLAUBERT_INPUTS_DOCSTRING = r"""
- `0` indicates the head is **masked**.
inputs_embeds (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the
@@ -164,8 +164,8 @@ FLAUBERT_INPUTS_DOCSTRING = r"""
more detail. This argument can be used only in eager mode, in graph mode the value in the config will be
used instead.
return_dict (`bool`, *optional*):
- Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This
- argument can be used in eager mode, in graph mode the value will always be set to True.
+ Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This argument can be used
+ in eager mode, in graph mode the value will always be set to True.
training (`bool`, *optional*, defaults to `False`):
Whether or not to use the model in training mode (some modules like dropout modules have different
behaviors between training and evaluation).
@@ -767,12 +767,13 @@ class TFFlaubertWithLMHeadModelOutput(ModelOutput):
logits (`tf.Tensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
diff --git a/src/transformers/models/flaubert/tokenization_flaubert.py b/src/transformers/models/flaubert/tokenization_flaubert.py
index ecdaaf874a..afe63e1953 100644
--- a/src/transformers/models/flaubert/tokenization_flaubert.py
+++ b/src/transformers/models/flaubert/tokenization_flaubert.py
@@ -82,12 +82,12 @@ class FlaubertTokenizer(XLMTokenizer):
- Moses preprocessing and tokenization.
- Normalizing all inputs text.
- - The arguments `special_tokens` and the function `set_special_tokens`, can be used to add additional symbols
- (like "__classify__") to a vocabulary.
+ - The arguments `special_tokens` and the function `set_special_tokens`, can be used to add additional symbols (like
+ "__classify__") to a vocabulary.
- The argument `do_lowercase` controls lower casing (automatically set for pretrained vocabularies).
- This tokenizer inherits from [`XLMTokenizer`]. Please check the superclass for usage examples
- and documentation regarding arguments.
+ This tokenizer inherits from [`XLMTokenizer`]. Please check the superclass for usage examples and documentation
+ regarding arguments.
"""
vocab_files_names = VOCAB_FILES_NAMES
diff --git a/src/transformers/models/fnet/configuration_fnet.py b/src/transformers/models/fnet/configuration_fnet.py
index 783064b5d4..5208e02e24 100644
--- a/src/transformers/models/fnet/configuration_fnet.py
+++ b/src/transformers/models/fnet/configuration_fnet.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" FNet model configuration """
+""" FNet model configuration"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging
@@ -29,19 +29,19 @@ FNET_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class FNetConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`FNetModel`]. It is used to
- instantiate an FNet model according to the specified arguments, defining the model architecture. Instantiating a
- configuration with the defaults will yield a similar configuration to that of the FNet [fnet-base](https://huggingface.co/google/fnet-base) architecture.
+ This is the configuration class to store the configuration of a [`FNetModel`]. It is used to instantiate an FNet
+ model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
+ defaults will yield a similar configuration to that of the FNet
+ [fnet-base](https://huggingface.co/google/fnet-base) architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
vocab_size (`int`, *optional*, defaults to 32000):
Vocabulary size of the FNet model. Defines the number of different tokens that can be represented by the
- `inputs_ids` passed when calling [`FNetModel`] or
- [`TFFNetModel`].
+ `inputs_ids` passed when calling [`FNetModel`] or [`TFFNetModel`].
hidden_size (`int`, *optional*, defaults to 768):
Dimension of the encoder layers and the pooler layer.
num_hidden_layers (`int`, *optional*, defaults to 12):
@@ -49,27 +49,26 @@ class FNetConfig(PretrainedConfig):
intermediate_size (`int`, *optional*, defaults to 3072):
Dimension of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
hidden_act (`str` or `function`, *optional*, defaults to `"gelu_new"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"selu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"selu"` and `"gelu_new"` are supported.
hidden_dropout_prob (`float`, *optional*, defaults to 0.1):
The dropout probabilitiy for all fully connected layers in the embeddings, encoder, and pooler.
max_position_embeddings (`int`, *optional*, defaults to 512):
The maximum sequence length that this model might ever be used with. Typically set this to something large
just in case (e.g., 512 or 1024 or 2048).
type_vocab_size (`int`, *optional*, defaults to 4):
- The vocabulary size of the `token_type_ids` passed when calling [`FNetModel`] or
- [`TFFNetModel`].
+ The vocabulary size of the `token_type_ids` passed when calling [`FNetModel`] or [`TFFNetModel`].
initializer_range (`float`, *optional*, defaults to 0.02):
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
layer_norm_eps (`float`, *optional*, defaults to 1e-12):
The epsilon used by the layer normalization layers.
use_tpu_fourier_optimizations (`bool`, *optional*, defaults to `False`):
- Determines whether to use TPU optimized FFTs. If `True`, the model will favor axis-wise FFTs
- transforms. Set to `False` for GPU/CPU hardware, in which case n-dimensional FFTs are used.
+ Determines whether to use TPU optimized FFTs. If `True`, the model will favor axis-wise FFTs transforms.
+ Set to `False` for GPU/CPU hardware, in which case n-dimensional FFTs are used.
tpu_short_seq_length (`int`, *optional*, defaults to 512):
The sequence length that is expected by the model when using TPUs. This will be used to initialize the DFT
- matrix only when *use_tpu_fourier_optimizations* is set to `True` and the input sequence is shorter
- than or equal to 4096 tokens.
+ matrix only when *use_tpu_fourier_optimizations* is set to `True` and the input sequence is shorter than or
+ equal to 4096 tokens.
Example:
diff --git a/src/transformers/models/fnet/modeling_fnet.py b/src/transformers/models/fnet/modeling_fnet.py
index e4930cfb8e..f76205558c 100755
--- a/src/transformers/models/fnet/modeling_fnet.py
+++ b/src/transformers/models/fnet/modeling_fnet.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" PyTorch FNet model. """
+""" PyTorch FNet model."""
import warnings
from dataclasses import dataclass
@@ -454,9 +454,9 @@ class FNetForPreTrainingOutput(ModelOutput):
Prediction scores of the next sequence prediction (classification) head (scores of True/False continuation
before SoftMax).
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of
- each layer plus the initial embedding outputs.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer
+ plus the initial embedding outputs.
"""
loss: Optional[torch.FloatTensor] = None
@@ -473,8 +473,7 @@ FNET_START_DOCSTRING = r"""
Parameters:
config ([`FNetConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
FNET_INPUTS_DOCSTRING = r"""
@@ -482,27 +481,28 @@ FNET_INPUTS_DOCSTRING = r"""
input_ids (`torch.LongTensor` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`FNetTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`FNetTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert *input_ids* indices into associated vectors
- than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_hidden_states (`bool`, *optional*):
Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
more detail.
@@ -519,8 +519,7 @@ class FNetModel(FNetPreTrainedModel):
"""
The model can behave as an encoder, following the architecture described in [FNet: Mixing Tokens with Fourier
- Transforms](https://arxiv.org/abs/2105.03824) by James Lee-Thorp, Joshua Ainslie, Ilya Eckstein, Santiago
- Ontanon.
+ Transforms](https://arxiv.org/abs/2105.03824) by James Lee-Thorp, Joshua Ainslie, Ilya Eckstein, Santiago Ontanon.
"""
@@ -656,8 +655,9 @@ class FNetForPreTraining(FNetPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
next_sentence_label (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair
(see `input_ids` docstring) Indices should be in `[0, 1]`:
@@ -714,7 +714,7 @@ class FNetForPreTraining(FNetPreTrainedModel):
)
-@add_start_docstrings("""FNet Model with a `language modeling` head on top. """, FNET_START_DOCSTRING)
+@add_start_docstrings("""FNet Model with a `language modeling` head on top.""", FNET_START_DOCSTRING)
class FNetForMaskedLM(FNetPreTrainedModel):
def __init__(self, config):
super().__init__(config)
@@ -750,8 +750,9 @@ class FNetForMaskedLM(FNetPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -780,7 +781,7 @@ class FNetForMaskedLM(FNetPreTrainedModel):
@add_start_docstrings(
- """FNet Model with a `next sentence prediction (classification)` head on top. """,
+ """FNet Model with a `next sentence prediction (classification)` head on top.""",
FNET_START_DOCSTRING,
)
class FNetForNextSentencePrediction(FNetPreTrainedModel):
@@ -907,8 +908,9 @@ class FNetForSequenceClassification(FNetPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -991,7 +993,8 @@ class FNetForMultipleChoice(FNetPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
+ Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
+ num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
`input_ids` above)
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1143,12 +1146,12 @@ class FNetForQuestionAnswering(FNetPreTrainedModel):
r"""
start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the start of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the end of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
diff --git a/src/transformers/models/fnet/tokenization_fnet.py b/src/transformers/models/fnet/tokenization_fnet.py
index ad7b1d6396..97e0865571 100644
--- a/src/transformers/models/fnet/tokenization_fnet.py
+++ b/src/transformers/models/fnet/tokenization_fnet.py
@@ -45,9 +45,10 @@ SPIECE_UNDERLINE = "▁"
class FNetTokenizer(PreTrainedTokenizer):
"""
- Construct an FNet tokenizer. Adapted from [`AlbertTokenizer`]. Based on [SentencePiece](https://github.com/google/sentencepiece). This tokenizer inherits from
- [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to this
- superclass for more information regarding those methods.
+ Construct an FNet tokenizer. Adapted from [`AlbertTokenizer`]. Based on
+ [SentencePiece](https://github.com/google/sentencepiece). This tokenizer inherits from [`PreTrainedTokenizer`]
+ which contains most of the main methods. Users should refer to this superclass for more information regarding those
+ methods.
Args:
vocab_file (`str`):
@@ -75,7 +76,9 @@ class FNetTokenizer(PreTrainedTokenizer):
The token used for masking values. This is the token used when training this model with masked language
modeling. This is the token which the model will try to predict.
sp_model_kwargs (`dict`, *optional*):
- Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things, to set:
+ Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for
+ SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things,
+ to set:
- `enable_sampling`: Enable subword regularization.
- `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout.
@@ -285,8 +288,7 @@ class FNetTokenizer(PreTrainedTokenizer):
Optional second list of IDs for sequence pairs.
Returns:
- `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given
- sequence(s).
+ `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
"""
sep = [self.sep_token_id]
cls = [self.cls_token_id]
diff --git a/src/transformers/models/fnet/tokenization_fnet_fast.py b/src/transformers/models/fnet/tokenization_fnet_fast.py
index d6a43f21e8..0c523a2259 100644
--- a/src/transformers/models/fnet/tokenization_fnet_fast.py
+++ b/src/transformers/models/fnet/tokenization_fnet_fast.py
@@ -55,9 +55,10 @@ SPIECE_UNDERLINE = "▁"
class FNetTokenizerFast(PreTrainedTokenizerFast):
"""
Construct a "fast" FNetTokenizer (backed by HuggingFace's *tokenizers* library). Adapted from
- [`AlbertTokenizerFast`]. Based on [Unigram](https://huggingface.co/docs/tokenizers/python/latest/components.html?highlight=unigram#models). This tokenizer
- inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should
- refer to this superclass for more information regarding those methods
+ [`AlbertTokenizerFast`]. Based on
+ [Unigram](https://huggingface.co/docs/tokenizers/python/latest/components.html?highlight=unigram#models). This
+ tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should refer to
+ this superclass for more information regarding those methods
Args:
vocab_file (`str`):
@@ -180,8 +181,7 @@ class FNetTokenizerFast(PreTrainedTokenizerFast):
Optional second list of IDs for sequence pairs.
Returns:
- `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given
- sequence(s).
+ `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
"""
sep = [self.sep_token_id]
cls = [self.cls_token_id]
diff --git a/src/transformers/models/fsmt/configuration_fsmt.py b/src/transformers/models/fsmt/configuration_fsmt.py
index 6e204a8c31..06939165ce 100644
--- a/src/transformers/models/fsmt/configuration_fsmt.py
+++ b/src/transformers/models/fsmt/configuration_fsmt.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" FSMT configuration """
+""" FSMT configuration"""
import copy
@@ -40,11 +40,11 @@ class DecoderConfig(PretrainedConfig):
class FSMTConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`FSMTModel`]. It is used to
- instantiate a FSMT model according to the specified arguments, defining the model architecture.
+ This is the configuration class to store the configuration of a [`FSMTModel`]. It is used to instantiate a FSMT
+ model according to the specified arguments, defining the model architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
langs (`List[str]`):
@@ -70,8 +70,8 @@ class FSMTConfig(PretrainedConfig):
encoder_ffn_dim (`int`, *optional*, defaults to 4096):
Dimensionality of the "intermediate" (often named feed-forward) layer in decoder.
activation_function (`str` or `Callable`, *optional*, defaults to `"relu"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"silu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"silu"` and `"gelu_new"` are supported.
dropout (`float`, *optional*, defaults to 0.1):
The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
attention_dropout (`float`, *optional*, defaults to 0.0):
@@ -102,13 +102,13 @@ class FSMTConfig(PretrainedConfig):
tie_word_embeddings (`bool`, *optional*, defaults to `False`):
Whether to tie input and output embeddings.
num_beams (`int`, *optional*, defaults to 5)
- Number of beams for beam search that will be used by default in the `generate` method of the model. 1
- means no beam search.
+ Number of beams for beam search that will be used by default in the `generate` method of the model. 1 means
+ no beam search.
length_penalty (`float`, *optional*, defaults to 1)
Exponential penalty to the length that will be used by default in the `generate` method of the model.
early_stopping (`bool`, *optional*, defaults to `False`)
- Flag that will be used by default in the `generate` method of the model. Whether to stop the beam
- search when at least `num_beams` sentences are finished per batch or not.
+ Flag that will be used by default in the `generate` method of the model. Whether to stop the beam search
+ when at least `num_beams` sentences are finished per batch or not.
use_cache (`bool`, *optional*, defaults to `True`):
Whether or not the model should return the last key/values attentions (not used by all models).
forced_eos_token_id (`int`, *optional*, defaults to 2):
diff --git a/src/transformers/models/fsmt/modeling_fsmt.py b/src/transformers/models/fsmt/modeling_fsmt.py
index 70294d269a..d93cb71d5e 100644
--- a/src/transformers/models/fsmt/modeling_fsmt.py
+++ b/src/transformers/models/fsmt/modeling_fsmt.py
@@ -182,19 +182,18 @@ PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py facebook/w
FSMT_START_DOCSTRING = r"""
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
Parameters:
config ([`FSMTConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
FSMT_GENERATION_EXAMPLE = r"""
@@ -202,16 +201,13 @@ FSMT_GENERATION_EXAMPLE = r"""
from transformers import FSMTTokenizer, FSMTForConditionalGeneration
- mname = "facebook/wmt19-ru-en"
- model = FSMTForConditionalGeneration.from_pretrained(mname)
- tokenizer = FSMTTokenizer.from_pretrained(mname)
+ mname = "facebook/wmt19-ru-en" model = FSMTForConditionalGeneration.from_pretrained(mname) tokenizer =
+ FSMTTokenizer.from_pretrained(mname)
- src_text = "Машинное обучение - это здорово, не так ли?"
- input_ids = tokenizer.encode(src_text, return_tensors='pt')
- outputs = model.generate(input_ids, num_beams=5, num_return_sequences=3)
- for i, output in enumerate(outputs):
- decoded = tokenizer.decode(output, skip_special_tokens=True)
- print(f"{i}: {decoded})
+ src_text = "Машинное обучение - это здорово, не так ли?" input_ids = tokenizer.encode(src_text,
+ return_tensors='pt') outputs = model.generate(input_ids, num_beams=5, num_return_sequences=3) for i, output in
+ enumerate(outputs):
+ decoded = tokenizer.decode(output, skip_special_tokens=True) print(f"{i}: {decoded})
# 1: Machine learning is great, isn't it? ...
"""
@@ -221,9 +217,8 @@ FSMT_INPUTS_DOCSTRING = r"""
input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
Indices of input sequence tokens in the vocabulary.
- IIndices can be obtained using [`FSTMTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ IIndices can be obtained using [`FSTMTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -236,18 +231,16 @@ FSMT_INPUTS_DOCSTRING = r"""
decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
Indices of decoder input sequence tokens in the vocabulary.
- Indices can be obtained using [`FSMTTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`FSMTTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are decoder input IDs?](../glossary#decoder-input-ids)
- FSMT uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If
- `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
- `past_key_values`).
+ FSMT uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values`
+ is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`).
decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
- Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will
- also be used by default.
+ Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
+ be used by default.
head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*):
Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`:
@@ -261,24 +254,24 @@ FSMT_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*):
- Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, 1]`:
+ Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0,
+ 1]`:
- 1 indicates the head is **not masked**,
- 0 indicates the head is **masked**.
encoder_outputs (`Tuple(torch.FloatTensor)`, *optional*):
- Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*:
- `attentions`) `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)` is a
- sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention of
- the decoder.
+ Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`)
+ `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)` is a sequence of hidden-states at
+ the output of the last layer of the encoder. Used in the cross-attention of the decoder.
past_key_values (`Tuple(torch.FloatTensor)` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
Contains precomputed key and value hidden-states of the attention blocks. Can be used to speed up decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ `decoder_input_ids` of shape `(batch_size, sequence_length)`.
use_cache (`bool`, *optional*, defaults to `True`):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`).
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`).
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -445,8 +438,7 @@ class EncoderLayer(nn.Module):
class FSMTEncoder(nn.Module):
"""
- Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer is a
- [`EncoderLayer`].
+ Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer is a [`EncoderLayer`].
Args:
config: FSMTConfig
@@ -491,8 +483,8 @@ class FSMTEncoder(nn.Module):
BaseModelOutput or Tuple comprised of:
- **x** (`torch.Tensor`): the last encoder layer's output of shape *(src_len, batch, embed_dim)*
- - **encoder_states** (`Tuple(torch.FloatTensor`)): all intermediate hidden states of shape
- *(src_len, batch, embed_dim)*. Only populated if *output_hidden_states:* is True.
+ - **encoder_states** (`Tuple(torch.FloatTensor`)): all intermediate hidden states of shape *(src_len,
+ batch, embed_dim)*. Only populated if *output_hidden_states:* is True.
- **all_attentions** (`Tuple(torch.FloatTensor`)): Attention weights for each layer.
During training might not be of length n_layers because of layer dropout.
"""
@@ -1153,7 +1145,8 @@ class FSMTForConditionalGeneration(PretrainedFSMTModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
+ Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
+ config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
(masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
Returns:
diff --git a/src/transformers/models/fsmt/tokenization_fsmt.py b/src/transformers/models/fsmt/tokenization_fsmt.py
index 73a1ca8322..2d11361439 100644
--- a/src/transformers/models/fsmt/tokenization_fsmt.py
+++ b/src/transformers/models/fsmt/tokenization_fsmt.py
@@ -140,12 +140,12 @@ class FSMTTokenizer(PreTrainedTokenizer):
- Moses preprocessing and tokenization.
- Normalizing all inputs text.
- - The arguments `special_tokens` and the function `set_special_tokens`, can be used to add additional symbols
- (like "__classify__") to a vocabulary.
+ - The arguments `special_tokens` and the function `set_special_tokens`, can be used to add additional symbols (like
+ "__classify__") to a vocabulary.
- The argument `langs` defines a pair of languages.
- This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods.
- Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
+ this superclass for more information regarding those methods.
Args:
langs (`List[str]`):
@@ -469,8 +469,7 @@ class FSMTTokenizer(PreTrainedTokenizer):
Optional second list of IDs for sequence pairs.
Returns:
- `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given
- sequence(s).
+ `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
Creates a mask from the two sequences passed to be used in a sequence-pair classification task. An
FAIRSEQ_TRANSFORMER sequence pair mask has the following format:
diff --git a/src/transformers/models/funnel/configuration_funnel.py b/src/transformers/models/funnel/configuration_funnel.py
index 5c2d1c962e..9496a63f48 100644
--- a/src/transformers/models/funnel/configuration_funnel.py
+++ b/src/transformers/models/funnel/configuration_funnel.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" Funnel Transformer model configuration """
+""" Funnel Transformer model configuration"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging
@@ -36,19 +36,18 @@ FUNNEL_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class FunnelConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`FunnelModel`] or a
- [`TFBertModel`]. It is used to instantiate a Funnel Transformer model according to the specified
- arguments, defining the model architecture. Instantiating a configuration with the defaults will yield a similar
- configuration to that of the Funnel Transformer [funnel-transformer/small](https://huggingface.co/funnel-transformer/small) architecture.
+ This is the configuration class to store the configuration of a [`FunnelModel`] or a [`TFBertModel`]. It is used to
+ instantiate a Funnel Transformer model according to the specified arguments, defining the model architecture.
+ Instantiating a configuration with the defaults will yield a similar configuration to that of the Funnel
+ Transformer [funnel-transformer/small](https://huggingface.co/funnel-transformer/small) architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
vocab_size (`int`, *optional*, defaults to 30522):
Vocabulary size of the Funnel transformer. Defines the number of different tokens that can be represented
- by the `inputs_ids` passed when calling [`FunnelModel`] or
- [`TFFunnelModel`].
+ by the `inputs_ids` passed when calling [`FunnelModel`] or [`TFFunnelModel`].
block_sizes (`List[int]`, *optional*, defaults to `[4, 4, 4]`):
The sizes of the blocks used in the model.
block_repeats (`List[int]`, *optional*):
@@ -64,8 +63,8 @@ class FunnelConfig(PretrainedConfig):
d_inner (`int`, *optional*, defaults to 3072):
Inner dimension in the feed-forward blocks.
hidden_act (`str` or `callable`, *optional*, defaults to `"gelu_new"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"silu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"silu"` and `"gelu_new"` are supported.
hidden_dropout (`float`, *optional*, defaults to 0.1):
The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
attention_dropout (`float`, *optional*, defaults to 0.1):
@@ -76,8 +75,7 @@ class FunnelConfig(PretrainedConfig):
The maximum sequence length that this model might ever be used with. Typically set this to something large
just in case (e.g., 512 or 1024 or 2048).
type_vocab_size (`int`, *optional*, defaults to 3):
- The vocabulary size of the `token_type_ids` passed when calling [`FunnelModel`] or
- [`TFFunnelModel`].
+ The vocabulary size of the `token_type_ids` passed when calling [`FunnelModel`] or [`TFFunnelModel`].
initializer_range (`float`, *optional*, defaults to 0.1):
The standard deviation of the *uniform initializer* for initializing all weight matrices in attention
layers.
@@ -90,8 +88,8 @@ class FunnelConfig(PretrainedConfig):
pooling_type (`str`, *optional*, defaults to `"mean"`):
Possible values are `"mean"` or `"max"`. The way pooling is performed at the beginning of each block.
attention_type (`str`, *optional*, defaults to `"relative_shift"`):
- Possible values are `"relative_shift"` or `"factorized"`. The former is faster on CPU/GPU while the
- latter is faster on TPU.
+ Possible values are `"relative_shift"` or `"factorized"`. The former is faster on CPU/GPU while the latter
+ is faster on TPU.
separate_cls (`bool`, *optional*, defaults to `True`):
Whether or not to separate the cls token when applying pooling.
truncate_seq (`bool`, *optional*, defaults to `False`):
diff --git a/src/transformers/models/funnel/modeling_funnel.py b/src/transformers/models/funnel/modeling_funnel.py
index b55c50f238..20c3aa1082 100644
--- a/src/transformers/models/funnel/modeling_funnel.py
+++ b/src/transformers/models/funnel/modeling_funnel.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" PyTorch Funnel Transformer model. """
+""" PyTorch Funnel Transformer model."""
import os
from dataclasses import dataclass
@@ -808,12 +808,13 @@ class FunnelForPreTrainingOutput(ModelOutput):
logits (`torch.FloatTensor` of shape `(batch_size, sequence_length)`):
Prediction scores of the head (scores for each token before SoftMax).
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -830,19 +831,18 @@ FUNNEL_START_DOCSTRING = r"""
The Funnel Transformer model was proposed in [Funnel-Transformer: Filtering out Sequential Redundancy for Efficient
Language Processing](https://arxiv.org/abs/2006.03236) by Zihang Dai, Guokun Lai, Yiming Yang, Quoc V. Le.
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
Parameters:
config ([`FunnelConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
FUNNEL_INPUTS_DOCSTRING = r"""
@@ -850,9 +850,8 @@ FUNNEL_INPUTS_DOCSTRING = r"""
input_ids (`torch.LongTensor` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`BertTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`BertTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
@@ -863,16 +862,17 @@ FUNNEL_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -1158,7 +1158,7 @@ class FunnelForPreTraining(FunnelPreTrainedModel):
)
-@add_start_docstrings("""Funnel Transformer Model with a `language modeling` head on top. """, FUNNEL_START_DOCSTRING)
+@add_start_docstrings("""Funnel Transformer Model with a `language modeling` head on top.""", FUNNEL_START_DOCSTRING)
class FunnelForMaskedLM(FunnelPreTrainedModel):
def __init__(self, config):
super().__init__(config)
@@ -1196,8 +1196,9 @@ class FunnelForMaskedLM(FunnelPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1269,8 +1270,9 @@ class FunnelForSequenceClassification(FunnelPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1359,7 +1361,8 @@ class FunnelForMultipleChoice(FunnelPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
+ Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
+ num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
`input_ids` above)
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1529,12 +1532,12 @@ class FunnelForQuestionAnswering(FunnelPreTrainedModel):
r"""
start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the start of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the end of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
diff --git a/src/transformers/models/funnel/modeling_tf_funnel.py b/src/transformers/models/funnel/modeling_tf_funnel.py
index ca77e070b3..373f8e192e 100644
--- a/src/transformers/models/funnel/modeling_tf_funnel.py
+++ b/src/transformers/models/funnel/modeling_tf_funnel.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" TF 2.0 Funnel model. """
+""" TF 2.0 Funnel model."""
import warnings
from dataclasses import dataclass
@@ -1010,12 +1010,13 @@ class TFFunnelForPreTrainingOutput(ModelOutput):
logits (`tf.Tensor` of shape `(batch_size, sequence_length)`):
Prediction scores of the head (scores for each token before SoftMax).
hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -1031,13 +1032,13 @@ FUNNEL_START_DOCSTRING = r"""
The Funnel Transformer model was proposed in [Funnel-Transformer: Filtering out Sequential Redundancy for Efficient
Language Processing](https://arxiv.org/abs/2006.03236) by Zihang Dai, Guokun Lai, Yiming Yang, Quoc V. Le.
- This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading or saving, resizing the input
- embeddings, pruning heads etc.)
+ This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use
- it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage
- and behavior.
+ This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it
+ as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and
+ behavior.
@@ -1046,11 +1047,11 @@ FUNNEL_START_DOCSTRING = r"""
- having all inputs as keyword arguments (like PyTorch models), or
- having all inputs as a list, tuple or dict in the first positional arguments.
- This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all
- the tensors in the first argument of the model call function: `model(inputs)`.
+ This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all the
+ tensors in the first argument of the model call function: `model(inputs)`.
- If you choose this second option, there are three possibilities you can use to gather all the input Tensors in
- the first positional argument :
+ If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the
+ first positional argument :
- a single Tensor with `input_ids` only and nothing else: `model(inputs_ids)`
- a list of varying length with one or several input Tensors IN THE ORDER given in the docstring:
@@ -1063,8 +1064,7 @@ FUNNEL_START_DOCSTRING = r"""
Parameters:
config ([`XxxConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
FUNNEL_INPUTS_DOCSTRING = r"""
@@ -1072,9 +1072,8 @@ FUNNEL_INPUTS_DOCSTRING = r"""
input_ids (`Numpy array` or `tf.Tensor` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`FunnelTokenizer`]. See
- [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for
- details.
+ Indices can be obtained using [`FunnelTokenizer`]. See [`PreTrainedTokenizer.__call__`] and
+ [`PreTrainedTokenizer.encode`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`Numpy array` or `tf.Tensor` of shape `({0})`, *optional*):
@@ -1085,16 +1084,17 @@ FUNNEL_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`Numpy array` or `tf.Tensor` of shape `({0})`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
inputs_embeds (`tf.Tensor` of shape `({0}, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the
@@ -1104,8 +1104,8 @@ FUNNEL_INPUTS_DOCSTRING = r"""
more detail. This argument can be used only in eager mode, in graph mode the value in the config will be
used instead.
return_dict (`bool`, *optional*):
- Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This
- argument can be used in eager mode, in graph mode the value will always be set to True.
+ Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This argument can be used
+ in eager mode, in graph mode the value will always be set to True.
training (`bool`, *optional*, defaults to `False`):
Whether or not to use the model in training mode (some modules like dropout modules have different
behaviors between training and evaluation).
@@ -1321,7 +1321,7 @@ class TFFunnelForPreTraining(TFFunnelPreTrainedModel):
return TFFunnelForPreTrainingOutput(logits=output.logits, hidden_states=hs, attentions=attns)
-@add_start_docstrings("""Funnel Model with a `language modeling` head on top. """, FUNNEL_START_DOCSTRING)
+@add_start_docstrings("""Funnel Model with a `language modeling` head on top.""", FUNNEL_START_DOCSTRING)
class TFFunnelForMaskedLM(TFFunnelPreTrainedModel, TFMaskedLanguageModelingLoss):
def __init__(self, config, *inputs, **kwargs):
super().__init__(config, *inputs, **kwargs)
@@ -1358,8 +1358,9 @@ class TFFunnelForMaskedLM(TFFunnelPreTrainedModel, TFMaskedLanguageModelingLoss)
):
r"""
labels (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
"""
inputs = input_processing(
func=self.call,
@@ -1446,8 +1447,9 @@ class TFFunnelForSequenceClassification(TFFunnelPreTrainedModel, TFSequenceClass
):
r"""
labels (`tf.Tensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
inputs = input_processing(
func=self.call,
@@ -1544,8 +1546,8 @@ class TFFunnelForMultipleChoice(TFFunnelPreTrainedModel, TFMultipleChoiceLoss):
):
r"""
labels (`tf.Tensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices]` where `num_choices` is the size of the second dimension of the input tensors. (See
- `input_ids` above)
+ Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices]`
+ where `num_choices` is the size of the second dimension of the input tensors. (See `input_ids` above)
"""
inputs = input_processing(
func=self.call,
@@ -1766,12 +1768,12 @@ class TFFunnelForQuestionAnswering(TFFunnelPreTrainedModel, TFQuestionAnsweringL
r"""
start_positions (`tf.Tensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the start of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
end_positions (`tf.Tensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the end of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
"""
inputs = input_processing(
func=self.call,
diff --git a/src/transformers/models/funnel/tokenization_funnel.py b/src/transformers/models/funnel/tokenization_funnel.py
index 991c048864..bb8b7548e9 100644
--- a/src/transformers/models/funnel/tokenization_funnel.py
+++ b/src/transformers/models/funnel/tokenization_funnel.py
@@ -59,11 +59,10 @@ class FunnelTokenizer(BertTokenizer):
r"""
Construct a Funnel Transformer tokenizer.
- [`FunnelTokenizer`] is identical to [`BertTokenizer`] and runs end-to-end
- tokenization: punctuation splitting and wordpiece.
+ [`FunnelTokenizer`] is identical to [`BertTokenizer`] and runs end-to-end tokenization: punctuation splitting and
+ wordpiece.
- Refer to superclass [`BertTokenizer`] for usage examples and documentation concerning
- parameters.
+ Refer to superclass [`BertTokenizer`] for usage examples and documentation concerning parameters.
"""
vocab_files_names = VOCAB_FILES_NAMES
@@ -127,8 +126,7 @@ class FunnelTokenizer(BertTokenizer):
Optional second list of IDs for sequence pairs.
Returns:
- `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given
- sequence(s).
+ `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
"""
sep = [self.sep_token_id]
cls = [self.cls_token_id]
diff --git a/src/transformers/models/funnel/tokenization_funnel_fast.py b/src/transformers/models/funnel/tokenization_funnel_fast.py
index a185f005ed..9fa7335ea5 100644
--- a/src/transformers/models/funnel/tokenization_funnel_fast.py
+++ b/src/transformers/models/funnel/tokenization_funnel_fast.py
@@ -72,11 +72,10 @@ class FunnelTokenizerFast(BertTokenizerFast):
r"""
Construct a "fast" Funnel Transformer tokenizer (backed by HuggingFace's *tokenizers* library).
- [`FunnelTokenizerFast`] is identical to [`BertTokenizerFast`] and runs
- end-to-end tokenization: punctuation splitting and wordpiece.
+ [`FunnelTokenizerFast`] is identical to [`BertTokenizerFast`] and runs end-to-end tokenization: punctuation
+ splitting and wordpiece.
- Refer to superclass [`BertTokenizerFast`] for usage examples and documentation concerning
- parameters.
+ Refer to superclass [`BertTokenizerFast`] for usage examples and documentation concerning parameters.
"""
vocab_files_names = VOCAB_FILES_NAMES
@@ -143,8 +142,7 @@ class FunnelTokenizerFast(BertTokenizerFast):
Optional second list of IDs for sequence pairs.
Returns:
- `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given
- sequence(s).
+ `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
"""
sep = [self.sep_token_id]
cls = [self.cls_token_id]
diff --git a/src/transformers/models/gpt2/configuration_gpt2.py b/src/transformers/models/gpt2/configuration_gpt2.py
index d119fb955b..a34c1d5c5c 100644
--- a/src/transformers/models/gpt2/configuration_gpt2.py
+++ b/src/transformers/models/gpt2/configuration_gpt2.py
@@ -13,7 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" OpenAI GPT-2 configuration """
+""" OpenAI GPT-2 configuration"""
from collections import OrderedDict
from typing import Any, List, Mapping, Optional
@@ -37,20 +37,19 @@ GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class GPT2Config(PretrainedConfig):
"""
- This is the configuration class to store the configuration of a [`GPT2Model`] or a
- [`TFGPT2Model`]. It is used to instantiate a GPT-2 model according to the specified arguments,
- defining the model architecture. Instantiating a configuration with the defaults will yield a similar configuration
- to that of the GPT-2 [small](https://huggingface.co/gpt2) architecture.
+ This is the configuration class to store the configuration of a [`GPT2Model`] or a [`TFGPT2Model`]. It is used to
+ instantiate a GPT-2 model according to the specified arguments, defining the model architecture. Instantiating a
+ configuration with the defaults will yield a similar configuration to that of the GPT-2
+ [small](https://huggingface.co/gpt2) architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
vocab_size (`int`, *optional*, defaults to 50257):
Vocabulary size of the GPT-2 model. Defines the number of different tokens that can be represented by the
- `inputs_ids` passed when calling [`GPT2Model`] or
- [`TFGPT2Model`].
+ `inputs_ids` passed when calling [`GPT2Model`] or [`TFGPT2Model`].
n_positions (`int`, *optional*, defaults to 1024):
The maximum sequence length that this model might ever be used with. Typically set this to something large
just in case (e.g., 512 or 1024 or 2048).
@@ -75,8 +74,8 @@ class GPT2Config(PretrainedConfig):
initializer_range (`float`, *optional*, defaults to 0.02):
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
summary_type (`string`, *optional*, defaults to `"cls_index"`):
- Argument used when doing sequence summary, used in the models [`GPT2DoubleHeadsModel`]
- and [`TFGPT2DoubleHeadsModel`].
+ Argument used when doing sequence summary, used in the models [`GPT2DoubleHeadsModel`] and
+ [`TFGPT2DoubleHeadsModel`].
Has to be one of the following options:
@@ -86,8 +85,8 @@ class GPT2Config(PretrainedConfig):
- `"cls_index"`: Supply a Tensor of classification token position (like GPT/GPT-2).
- `"attn"`: Not implemented now, use multi-head attention.
summary_use_proj (`bool`, *optional*, defaults to `True`):
- Argument used when doing sequence summary, used in the models [`GPT2DoubleHeadsModel`]
- and [`TFGPT2DoubleHeadsModel`].
+ Argument used when doing sequence summary, used in the models [`GPT2DoubleHeadsModel`] and
+ [`TFGPT2DoubleHeadsModel`].
Whether or not to add a projection after the vector extraction.
summary_activation (`str`, *optional*):
@@ -96,13 +95,13 @@ class GPT2Config(PretrainedConfig):
Pass `"tanh"` for a tanh activation to the output, any other value will result in no activation.
summary_proj_to_labels (`bool`, *optional*, defaults to `True`):
- Argument used when doing sequence summary, used in the models [`GPT2DoubleHeadsModel`]
- and [`TFGPT2DoubleHeadsModel`].
+ Argument used when doing sequence summary, used in the models [`GPT2DoubleHeadsModel`] and
+ [`TFGPT2DoubleHeadsModel`].
Whether the projection outputs should have `config.num_labels` or `config.hidden_size` classes.
summary_first_dropout (`float`, *optional*, defaults to 0.1):
- Argument used when doing sequence summary, used in the models [`GPT2DoubleHeadsModel`]
- and [`TFGPT2DoubleHeadsModel`].
+ Argument used when doing sequence summary, used in the models [`GPT2DoubleHeadsModel`] and
+ [`TFGPT2DoubleHeadsModel`].
The dropout ratio to be used after the projection and activation.
scale_attn_weights (`bool`, *optional*, defaults to `True`):
diff --git a/src/transformers/models/gpt2/modeling_flax_gpt2.py b/src/transformers/models/gpt2/modeling_flax_gpt2.py
index c3931a17de..c7867a6400 100644
--- a/src/transformers/models/gpt2/modeling_flax_gpt2.py
+++ b/src/transformers/models/gpt2/modeling_flax_gpt2.py
@@ -42,12 +42,13 @@ _TOKENIZER_FOR_DOC = "GPT2Tokenizer"
GPT2_START_DOCSTRING = r"""
- This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading or saving, resizing the input
- embeddings, pruning heads etc.)
+ This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a Flax Linen [flax.nn.Module](https://flax.readthedocs.io/en/latest/_autosummary/flax.nn.module.html) subclass. Use it as a regular Flax
- Module and refer to the Flax documentation for all matter related to general usage and behavior.
+ This model is also a Flax Linen
+ [flax.nn.Module](https://flax.readthedocs.io/en/latest/_autosummary/flax.nn.module.html) subclass. Use it as a
+ regular Flax Module and refer to the Flax documentation for all matter related to general usage and behavior.
Finally, this model supports inherent JAX features such as:
@@ -59,11 +60,10 @@ GPT2_START_DOCSTRING = r"""
Parameters:
config ([`GPT2Config`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the
- model weights.
+ configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the model weights.
dtype (`jax.numpy.dtype`, *optional*, defaults to `jax.numpy.float32`):
- The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on
- GPUs) and `jax.numpy.bfloat16` (on TPUs).
+ The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on GPUs) and
+ `jax.numpy.bfloat16` (on TPUs).
This can be used to enable mixed-precision training or half-precision inference on GPUs or TPUs. If
specified all the computation will be performed with the given `dtype`.
@@ -71,8 +71,8 @@ GPT2_START_DOCSTRING = r"""
**Note that this only specifies the dtype of the computation and does not influence the dtype of model
parameters.**
- If you wish to change the dtype of the model parameters, see
- [`~FlaxPreTrainedModel.to_fp16`] and [`~FlaxPreTrainedModel.to_bf16`].
+ If you wish to change the dtype of the model parameters, see [`~FlaxPreTrainedModel.to_fp16`] and
+ [`~FlaxPreTrainedModel.to_bf16`].
"""
GPT2_INPUTS_DOCSTRING = r"""
@@ -80,9 +80,8 @@ GPT2_INPUTS_DOCSTRING = r"""
input_ids (`numpy.ndarray` of shape `(batch_size, input_ids_length)`):
`input_ids_length` = `sequence_length`. Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`GPT2Tokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`GPT2Tokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`numpy.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
@@ -93,7 +92,8 @@ GPT2_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
position_ids (`numpy.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
past_key_values (`Dict[str, np.ndarray]`, *optional*, returned by `init_cache` or when passing previous `past_key_values`):
Dictionary of pre-computed hidden-states (key and values in the attention blocks) that can be used for fast
auto-regressive decoding. Pre-computed key and value hidden-states are of shape *[batch_size, max_length]*.
diff --git a/src/transformers/models/gpt2/modeling_gpt2.py b/src/transformers/models/gpt2/modeling_gpt2.py
index 46f8f9a29d..7a4a27c7ed 100644
--- a/src/transformers/models/gpt2/modeling_gpt2.py
+++ b/src/transformers/models/gpt2/modeling_gpt2.py
@@ -503,17 +503,19 @@ class GPT2DoubleHeadsModelOutput(ModelOutput):
mc_logits (`torch.FloatTensor` of shape `(batch_size, num_choices)`):
Prediction scores of the multiple choice classification head (scores for each choice before SoftMax).
past_key_values (`Tuple[Tuple[torch.Tensor]]`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- Tuple of length `config.n_layers`, containing tuples of tensors of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`).
+ Tuple of length `config.n_layers`, containing tuples of tensors of shape `(batch_size, num_heads,
+ sequence_length, embed_size_per_head)`).
Contains pre-computed hidden-states (key and values in the attention blocks) that can be used (see
`past_key_values` input) to speed up sequential decoding.
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
GPT2Attentions weights after the attention softmax, used to compute the weighted average in the
self-attention heads.
@@ -530,19 +532,18 @@ class GPT2DoubleHeadsModelOutput(ModelOutput):
GPT2_START_DOCSTRING = r"""
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
Parameters:
config ([`GPT2Config`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
GPT2_INPUTS_DOCSTRING = r"""
@@ -552,19 +553,17 @@ GPT2_INPUTS_DOCSTRING = r"""
`past_key_values[0][0].shape[-2]` (`sequence_length` of input past key value states). Indices of input
sequence tokens in the vocabulary.
- If `past_key_values` is used, only `input_ids` that do not have their past calculated should be
- passed as `input_ids`.
+ If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as
+ `input_ids`.
- Indices can be obtained using [`GPT2Tokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`GPT2Tokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
past_key_values (`Tuple[Tuple[torch.Tensor]]` of length `config.n_layers`):
Contains precomputed hidden-states (key and values in the attention blocks) as computed by the model (see
- `past_key_values` output below). Can be used to speed up sequential decoding. The `input_ids` which
- have their past given to this model should not be passed as `input_ids` as they have already been
- computed.
+ `past_key_values` output below). Can be used to speed up sequential decoding. The `input_ids` which have
+ their past given to this model should not be passed as `input_ids` as they have already been computed.
attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
@@ -573,14 +572,16 @@ GPT2_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
@@ -590,15 +591,15 @@ GPT2_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
If `past_key_values` is used, optionally only the last `inputs_embeds` have to be input (see
`past_key_values`).
use_cache (`bool`, *optional*):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`).
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`).
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -1036,8 +1037,8 @@ class GPT2LMHeadModel(GPT2PreTrainedModel):
r"""
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
- `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to
- `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
+ `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
+ are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1090,9 +1091,9 @@ class GPT2LMHeadModel(GPT2PreTrainedModel):
@staticmethod
def _reorder_cache(past: Tuple[Tuple[torch.Tensor]], beam_idx: torch.Tensor) -> Tuple[Tuple[torch.Tensor]]:
"""
- This function is used to re-order the `past_key_values` cache if
- [`~PreTrainedModel.beam_search`] or [`~PreTrainedModel.beam_sample`] is
- called. This is required to match `past_key_values` with the correct beam_idx at every generation step.
+ This function is used to re-order the `past_key_values` cache if [`~PreTrainedModel.beam_search`] or
+ [`~PreTrainedModel.beam_sample`] is called. This is required to match `past_key_values` with the correct
+ beam_idx at every generation step.
"""
return tuple(
tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past)
@@ -1205,14 +1206,15 @@ class GPT2DoubleHeadsModel(GPT2PreTrainedModel):
):
r"""
mc_token_ids (`torch.LongTensor` of shape `(batch_size, num_choices)`, *optional*, default to index of the last token of the input):
- Index of the classification token in each input sequence. Selected in the range `[0, input_ids.size(-1) - 1[`.
+ Index of the classification token in each input sequence. Selected in the range `[0, input_ids.size(-1) -
+ 1[`.
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
`labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size - 1]` All labels set to
`-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size - 1]`
mc_labels (`torch.LongTensor` of shape `(batch_size)`, *optional*):
- Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices]` where *num_choices* is the size of the second dimension of the input tensors. (see
- *input_ids* above)
+ Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices]`
+ where *num_choices* is the size of the second dimension of the input tensors. (see *input_ids* above)
Return:
@@ -1297,9 +1299,9 @@ class GPT2DoubleHeadsModel(GPT2PreTrainedModel):
@staticmethod
def _reorder_cache(past: Tuple[Tuple[torch.Tensor]], beam_idx: torch.Tensor) -> Tuple[Tuple[torch.Tensor]]:
"""
- This function is used to re-order the `past_key_values` cache if
- [`~PreTrainedModel.beam_search`] or [`~PreTrainedModel.beam_sample`] is
- called. This is required to match `past_key_values` with the correct beam_idx at every generation step.
+ This function is used to re-order the `past_key_values` cache if [`~PreTrainedModel.beam_search`] or
+ [`~PreTrainedModel.beam_sample`] is called. This is required to match `past_key_values` with the correct
+ beam_idx at every generation step.
"""
return tuple(
tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past)
@@ -1311,14 +1313,14 @@ class GPT2DoubleHeadsModel(GPT2PreTrainedModel):
"""
The GPT2 Model transformer with a sequence classification head on top (linear layer).
- [`GPT2ForSequenceClassification`] uses the last token in order to do the classification, as
- other causal models (e.g. GPT-1) do.
+ [`GPT2ForSequenceClassification`] uses the last token in order to do the classification, as other causal models
+ (e.g. GPT-1) do.
Since it does classification on the last token, it requires to know the position of the last token. If a
- `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each
- row. If no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot
- guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take
- the last value in each row of the batch).
+ `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
+ no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
+ padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
+ each row of the batch).
""",
GPT2_START_DOCSTRING,
)
@@ -1362,8 +1364,9 @@ class GPT2ForSequenceClassification(GPT2PreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1493,8 +1496,9 @@ class GPT2ForTokenClassification(GPT2PreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
diff --git a/src/transformers/models/gpt2/modeling_tf_gpt2.py b/src/transformers/models/gpt2/modeling_tf_gpt2.py
index 568ba30e50..1bac2f6226 100644
--- a/src/transformers/models/gpt2/modeling_tf_gpt2.py
+++ b/src/transformers/models/gpt2/modeling_tf_gpt2.py
@@ -13,7 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" TF 2.0 OpenAI GPT-2 model. """
+""" TF 2.0 OpenAI GPT-2 model."""
from dataclasses import dataclass
from typing import List, Optional, Tuple
@@ -595,17 +595,19 @@ class TFGPT2DoubleHeadsModelOutput(ModelOutput):
mc_logits (`tf.Tensor` of shape `(batch_size, num_choices)`):
Prediction scores of the multiple choice classification head (scores for each choice before SoftMax).
past_key_values (`List[tf.Tensor]`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- List of `tf.Tensor` of length `config.n_layers`, with each tensor of shape `(2, batch_size, num_heads, sequence_length, embed_size_per_head)`).
+ List of `tf.Tensor` of length `config.n_layers`, with each tensor of shape `(2, batch_size, num_heads,
+ sequence_length, embed_size_per_head)`).
Contains pre-computed hidden-states (key and values in the attention blocks) that can be used (see
`past_key_values` input) to speed up sequential decoding.
hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -620,13 +622,13 @@ class TFGPT2DoubleHeadsModelOutput(ModelOutput):
GPT2_START_DOCSTRING = r"""
- This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading or saving, resizing the input
- embeddings, pruning heads etc.)
+ This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use
- it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage
- and behavior.
+ This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it
+ as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and
+ behavior.
@@ -635,11 +637,11 @@ GPT2_START_DOCSTRING = r"""
- having all inputs as keyword arguments (like PyTorch models), or
- having all inputs as a list, tuple or dict in the first positional arguments.
- This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all
- the tensors in the first argument of the model call function: `model(inputs)`.
+ This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all the
+ tensors in the first argument of the model call function: `model(inputs)`.
- If you choose this second option, there are three possibilities you can use to gather all the input Tensors in
- the first positional argument :
+ If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the
+ first positional argument :
- a single Tensor with `input_ids` only and nothing else: `model(inputs_ids)`
- a list of varying length with one or several input Tensors IN THE ORDER given in the docstring:
@@ -652,22 +654,19 @@ GPT2_START_DOCSTRING = r"""
Parameters:
config ([`GPT2Config`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
GPT2_INPUTS_DOCSTRING = r"""
Args:
input_ids (`Numpy array` or `tf.Tensor` of shape `(batch_size, input_ids_length)`):
- `input_ids_length` = `sequence_length` if `past` is `None` else `past[0].shape[-2]`
- (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary.
+ `input_ids_length` = `sequence_length` if `past` is `None` else `past[0].shape[-2]` (`sequence_length` of
+ input past key value states). Indices of input sequence tokens in the vocabulary.
- If `past` is used, only input IDs that do not have their past calculated should be passed as
- `input_ids`.
+ If `past` is used, only input IDs that do not have their past calculated should be passed as `input_ids`.
- Indices can be obtained using [`GPT2Tokenizer`]. See
- [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for
- details.
+ Indices can be obtained using [`GPT2Tokenizer`]. See [`PreTrainedTokenizer.__call__`] and
+ [`PreTrainedTokenizer.encode`] for details.
[What are input IDs?](../glossary#input-ids)
past (`List[tf.Tensor]` of length `config.n_layers`):
@@ -682,14 +681,16 @@ GPT2_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`tf.Tensor` or `Numpy array` of shape `(batch_size, sequence_length)`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`tf.Tensor` or `Numpy array` of shape `(batch_size, sequence_length)`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
head_mask (`Numpy array` or `tf.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
@@ -699,9 +700,9 @@ GPT2_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
inputs_embeds (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the
@@ -711,8 +712,8 @@ GPT2_INPUTS_DOCSTRING = r"""
more detail. This argument can be used only in eager mode, in graph mode the value in the config will be
used instead.
return_dict (`bool`, *optional*):
- Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This
- argument can be used in eager mode, in graph mode the value will always be set to True.
+ Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This argument can be used
+ in eager mode, in graph mode the value will always be set to True.
training (`bool`, *optional*, defaults to `False`):
Whether or not to use the model in training mode (some modules like dropout modules have different
behaviors between training and evaluation).
@@ -766,12 +767,12 @@ class TFGPT2Model(TFGPT2PreTrainedModel):
past (`Tuple[Tuple[tf.Tensor]]` of length `config.n_layers`)
contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
- If `past` are used, the user can optionally input only the last `decoder_input_ids` (those that
- don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ If `past` are used, the user can optionally input only the last `decoder_input_ids` (those that don't have
+ their past key value states given to this model) of shape `(batch_size, 1)` instead of all
`decoder_input_ids` of shape `(batch_size, sequence_length)`.
use_cache (`bool`, *optional*, defaults to `True`):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past`). Set to `False` during training, `True` during generation
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past`). Set to `False` during training, `True` during generation
"""
inputs = input_processing(
func=self.call,
@@ -896,14 +897,15 @@ class TFGPT2LMHeadModel(TFGPT2PreTrainedModel, TFCausalLanguageModelingLoss):
past (`Tuple[Tuple[tf.Tensor]]` of length `config.n_layers`)
contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
- If `past` are used, the user can optionally input only the last `decoder_input_ids` (those that
- don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ If `past` are used, the user can optionally input only the last `decoder_input_ids` (those that don't have
+ their past key value states given to this model) of shape `(batch_size, 1)` instead of all
`decoder_input_ids` of shape `(batch_size, sequence_length)`.
use_cache (`bool`, *optional*, defaults to `True`):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past`). Set to `False` during training, `True` during generation
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past`). Set to `False` during training, `True` during generation
labels (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the cross entropy classification loss. Indices should be in `[0, ..., config.vocab_size - 1]`.
+ Labels for computing the cross entropy classification loss. Indices should be in `[0, ...,
+ config.vocab_size - 1]`.
"""
inputs = input_processing(
func=self.call,
@@ -1020,7 +1022,8 @@ class TFGPT2DoubleHeadsModel(TFGPT2PreTrainedModel):
):
r"""
mc_token_ids (`tf.Tensor` or `Numpy array` of shape `(batch_size, num_choices)`, *optional*, default to index of the last token of the input):
- Index of the classification token in each input sequence. Selected in the range `[0, input_ids.size(-1) - 1[`.
+ Index of the classification token in each input sequence. Selected in the range `[0, input_ids.size(-1) -
+ 1[`.
Return:
@@ -1148,14 +1151,14 @@ class TFGPT2DoubleHeadsModel(TFGPT2PreTrainedModel):
"""
The GPT2 Model transformer with a sequence classification head on top (linear layer).
- [`TFGPT2ForSequenceClassification`] uses the last token in order to do the classification, as
- other causal models (e.g. GPT-1) do.
+ [`TFGPT2ForSequenceClassification`] uses the last token in order to do the classification, as other causal models
+ (e.g. GPT-1) do.
Since it does classification on the last token, it requires to know the position of the last token. If a
- `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each
- row. If no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot
- guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take
- the last value in each row of the batch).
+ `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
+ no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
+ padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
+ each row of the batch).
""",
GPT2_START_DOCSTRING,
)
@@ -1197,7 +1200,8 @@ class TFGPT2ForSequenceClassification(TFGPT2PreTrainedModel, TFSequenceClassific
):
r"""
labels (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the cross entropy classification loss. Indices should be in `[0, ..., config.vocab_size - 1]`.
+ Labels for computing the cross entropy classification loss. Indices should be in `[0, ...,
+ config.vocab_size - 1]`.
"""
inputs = input_processing(
func=self.call,
diff --git a/src/transformers/models/gpt2/tokenization_gpt2.py b/src/transformers/models/gpt2/tokenization_gpt2.py
index e334c6c7c9..0c4cf69f77 100644
--- a/src/transformers/models/gpt2/tokenization_gpt2.py
+++ b/src/transformers/models/gpt2/tokenization_gpt2.py
@@ -122,13 +122,12 @@ class GPT2Tokenizer(PreTrainedTokenizer):
- When used with `is_split_into_words=True`, this tokenizer will add a space before each word (even the first
- one).
+ When used with `is_split_into_words=True`, this tokenizer will add a space before each word (even the first one).
- This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods.
- Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
+ this superclass for more information regarding those methods.
Args:
vocab_file (`str`):
@@ -136,7 +135,8 @@ class GPT2Tokenizer(PreTrainedTokenizer):
merges_file (`str`):
Path to the merges file.
errors (`str`, *optional*, defaults to `"replace"`):
- Paradigm to follow when decoding bytes to UTF-8. See [bytes.decode](https://docs.python.org/3/library/stdtypes.html#bytes.decode) for more information.
+ Paradigm to follow when decoding bytes to UTF-8. See
+ [bytes.decode](https://docs.python.org/3/library/stdtypes.html#bytes.decode) for more information.
unk_token (`str`, *optional*, defaults to `<|endoftext|>`):
The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
token instead.
diff --git a/src/transformers/models/gpt2/tokenization_gpt2_fast.py b/src/transformers/models/gpt2/tokenization_gpt2_fast.py
index 54ea4fa27e..e244a5d21e 100644
--- a/src/transformers/models/gpt2/tokenization_gpt2_fast.py
+++ b/src/transformers/models/gpt2/tokenization_gpt2_fast.py
@@ -89,13 +89,12 @@ class GPT2TokenizerFast(PreTrainedTokenizerFast):
- When used with `is_split_into_words=True`, this tokenizer needs to be instantiated with
- `add_prefix_space=True`.
+ When used with `is_split_into_words=True`, this tokenizer needs to be instantiated with `add_prefix_space=True`.
- This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main
- methods. Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should
+ refer to this superclass for more information regarding those methods.
Args:
vocab_file (`str`):
@@ -103,7 +102,8 @@ class GPT2TokenizerFast(PreTrainedTokenizerFast):
merges_file (`str`):
Path to the merges file.
errors (`str`, *optional*, defaults to `"replace"`):
- Paradigm to follow when decoding bytes to UTF-8. See [bytes.decode](https://docs.python.org/3/library/stdtypes.html#bytes.decode) for more information.
+ Paradigm to follow when decoding bytes to UTF-8. See
+ [bytes.decode](https://docs.python.org/3/library/stdtypes.html#bytes.decode) for more information.
unk_token (`str`, *optional*, defaults to `<|endoftext|>`):
The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
token instead.
diff --git a/src/transformers/models/gpt_neo/configuration_gpt_neo.py b/src/transformers/models/gpt_neo/configuration_gpt_neo.py
index 3b40bd72b5..1e453de3f8 100644
--- a/src/transformers/models/gpt_neo/configuration_gpt_neo.py
+++ b/src/transformers/models/gpt_neo/configuration_gpt_neo.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" GPT Neo model configuration """
+""" GPT Neo model configuration"""
from collections import OrderedDict
from typing import Any, Mapping, Optional
@@ -33,23 +33,24 @@ GPT_NEO_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class GPTNeoConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`GPTNeoModel`]. It is used to
- instantiate a GPT Neo model according to the specified arguments, defining the model architecture. Instantiating a
- configuration with the defaults will yield a similar configuration to that of the GPTNeo [gpt-neo-1.3B](https://huggingface.co/EleutherAI/gpt-neo-1.3B) architecture.
+ This is the configuration class to store the configuration of a [`GPTNeoModel`]. It is used to instantiate a GPT
+ Neo model according to the specified arguments, defining the model architecture. Instantiating a configuration with
+ the defaults will yield a similar configuration to that of the GPTNeo
+ [gpt-neo-1.3B](https://huggingface.co/EleutherAI/gpt-neo-1.3B) architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
vocab_size (`int`, *optional*, defaults to 50257):
Vocabulary size of the GPT Neo model. Defines the number of different tokens that can be represented by the
- `inputs_ids` passed when calling [`GPTNeoModel`]. Vocabulary size of the model.
- Defines the different tokens that can be represented by the *inputs_ids* passed to the forward method of
- [`GPTNeoModel`].
+ `inputs_ids` passed when calling [`GPTNeoModel`]. Vocabulary size of the model. Defines the different
+ tokens that can be represented by the *inputs_ids* passed to the forward method of [`GPTNeoModel`].
attention_types (`List`, *optional*, defaults to `[[["global", "local"], 12]]`):
- The type of attention for each layer in a `List` of the following format `[[["attention_type"], num_layerss]]` e.g. for a 24 layer model `[[["global"], 24]]` or `[[["global", "local"], 12]]`
- Choose the value of `attention_type` from `["global", "local"]`
+ The type of attention for each layer in a `List` of the following format `[[["attention_type"],
+ num_layerss]]` e.g. for a 24 layer model `[[["global"], 24]]` or `[[["global", "local"], 12]]` Choose the
+ value of `attention_type` from `["global", "local"]`
hidden_size (`int`, *optional*, defaults to 2048):
Dimensionality of the encoder layers and the pooler layer.
num_layers (`int`, *optional*, defaults to 24):
@@ -59,8 +60,8 @@ class GPTNeoConfig(PretrainedConfig):
intermediate_size (`int`, *optional*, defaults to 8192):
Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
activation_function (`str` or `function`, *optional*, defaults to `"gelu_new"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"selu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"selu"` and `"gelu_new"` are supported.
embed_dropout (`float`, *optional*, defaults to 0.0):
The dropout probabilitiy for all fully connected layers in the embeddings, encoder, and pooler.
attention_dropout (`float`, *optional*, defaults to 0.0):
diff --git a/src/transformers/models/gpt_neo/modeling_flax_gpt_neo.py b/src/transformers/models/gpt_neo/modeling_flax_gpt_neo.py
index 17e042dec8..05779a259f 100644
--- a/src/transformers/models/gpt_neo/modeling_flax_gpt_neo.py
+++ b/src/transformers/models/gpt_neo/modeling_flax_gpt_neo.py
@@ -40,12 +40,13 @@ _CHECKPOINT_FOR_DOC = "EleutherAI/gpt-neo-1.3B"
GPT_NEO_START_DOCSTRING = r"""
- This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading or saving, resizing the input
- embeddings, pruning heads etc.)
+ This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a Flax Linen [flax.nn.Module](https://flax.readthedocs.io/en/latest/_autosummary/flax.nn.module.html) subclass. Use it as a regular Flax
- Module and refer to the Flax documentation for all matter related to general usage and behavior.
+ This model is also a Flax Linen
+ [flax.nn.Module](https://flax.readthedocs.io/en/latest/_autosummary/flax.nn.module.html) subclass. Use it as a
+ regular Flax Module and refer to the Flax documentation for all matter related to general usage and behavior.
Finally, this model supports inherent JAX features such as:
@@ -57,11 +58,10 @@ GPT_NEO_START_DOCSTRING = r"""
Parameters:
config ([`GPTNeoConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the
- model weights.
+ configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the model weights.
dtype (`jax.numpy.dtype`, *optional*, defaults to `jax.numpy.float32`):
- The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on
- GPUs) and `jax.numpy.bfloat16` (on TPUs).
+ The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on GPUs) and
+ `jax.numpy.bfloat16` (on TPUs).
This can be used to enable mixed-precision training or half-precision inference on GPUs or TPUs. If
specified all the computation will be performed with the given `dtype`.
@@ -69,8 +69,8 @@ GPT_NEO_START_DOCSTRING = r"""
**Note that this only specifies the dtype of the computation and does not influence the dtype of model
parameters.**
- If you wish to change the dtype of the model parameters, see
- [`~FlaxPreTrainedModel.to_fp16`] and [`~FlaxPreTrainedModel.to_bf16`].
+ If you wish to change the dtype of the model parameters, see [`~FlaxPreTrainedModel.to_fp16`] and
+ [`~FlaxPreTrainedModel.to_bf16`].
"""
GPT_NEO_INPUTS_DOCSTRING = r"""
@@ -78,9 +78,8 @@ GPT_NEO_INPUTS_DOCSTRING = r"""
input_ids (`numpy.ndarray` of shape `(batch_size, input_ids_length)`):
`input_ids_length` = `sequence_length`. Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`GPTNeoTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`GPTNeoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`numpy.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
@@ -91,7 +90,8 @@ GPT_NEO_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
position_ids (`numpy.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
past_key_values (`Dict[str, np.ndarray]`, *optional*, returned by `init_cache` or when passing previous `past_key_values`):
Dictionary of pre-computed hidden-states (key and values in the attention blocks) that can be used for fast
auto-regressive decoding. Pre-computed key and value hidden-states are of shape *[batch_size, max_length]*.
diff --git a/src/transformers/models/gpt_neo/modeling_gpt_neo.py b/src/transformers/models/gpt_neo/modeling_gpt_neo.py
index 13b3278f53..7176cfa790 100755
--- a/src/transformers/models/gpt_neo/modeling_gpt_neo.py
+++ b/src/transformers/models/gpt_neo/modeling_gpt_neo.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" PyTorch GPT Neo model. """
+""" PyTorch GPT Neo model."""
import os
@@ -389,19 +389,18 @@ class GPTNeoPreTrainedModel(PreTrainedModel):
GPT_NEO_START_DOCSTRING = r"""
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
Parameters:
config ([`GPTNeoConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
GPT_NEO_INPUTS_DOCSTRING = r"""
@@ -411,19 +410,17 @@ GPT_NEO_INPUTS_DOCSTRING = r"""
`past_key_values[0][0].shape[-2]` (`sequence_length` of input past key value states). Indices of input
sequence tokens in the vocabulary.
- If `past_key_values` is used, only `input_ids` that do not have their past calculated should be
- passed as `input_ids`.
+ If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as
+ `input_ids`.
- Indices can be obtained using [`GPTNeoTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`GPTNeoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
past_key_values (`Tuple[Tuple[torch.Tensor]]` of length `config.num_layers`):
Contains precomputed hidden-states (key and values in the attention blocks) as computed by the model (see
- `past_key_values` output below). Can be used to speed up sequential decoding. The `input_ids` which
- have their past given to this model should not be passed as `input_ids` as they have already been
- computed.
+ `past_key_values` output below). Can be used to speed up sequential decoding. The `input_ids` which have
+ their past given to this model should not be passed as `input_ids` as they have already been computed.
attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
@@ -432,14 +429,16 @@ GPT_NEO_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
@@ -449,15 +448,15 @@ GPT_NEO_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
If `past_key_values` is used, optionally only the last `inputs_embeds` have to be input (see
`past_key_values`).
use_cache (`bool`, *optional*):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`).
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`).
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -736,8 +735,8 @@ class GPTNeoForCausalLM(GPTNeoPreTrainedModel):
r"""
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
- `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to
- `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
+ `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
+ are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -789,9 +788,9 @@ class GPTNeoForCausalLM(GPTNeoPreTrainedModel):
@staticmethod
def _reorder_cache(past: Tuple[Tuple[torch.Tensor]], beam_idx: torch.Tensor) -> Tuple[Tuple[torch.Tensor]]:
"""
- This function is used to re-order the `past_key_values` cache if
- [`~PretrainedModel.beam_search`] or [`~PretrainedModel.beam_sample`] is
- called. This is required to match `past_key_values` with the correct beam_idx at every generation step.
+ This function is used to re-order the `past_key_values` cache if [`~PretrainedModel.beam_search`] or
+ [`~PretrainedModel.beam_sample`] is called. This is required to match `past_key_values` with the correct
+ beam_idx at every generation step.
"""
return tuple(
tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past)
@@ -803,14 +802,14 @@ class GPTNeoForCausalLM(GPTNeoPreTrainedModel):
"""
The GPTNeo Model transformer with a sequence classification head on top (linear layer).
- [`GPTNeoForSequenceClassification`] uses the last token in order to do the classification, as
- other causal models (e.g. GPT-1) do.
+ [`GPTNeoForSequenceClassification`] uses the last token in order to do the classification, as other causal models
+ (e.g. GPT-1) do.
Since it does classification on the last token, it requires to know the position of the last token. If a
- `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each
- row. If no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot
- guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take
- the last value in each row of the batch).
+ `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
+ no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
+ padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
+ each row of the batch).
""",
GPT_NEO_START_DOCSTRING,
)
@@ -850,8 +849,9 @@ class GPTNeoForSequenceClassification(GPTNeoPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
diff --git a/src/transformers/models/gptj/configuration_gptj.py b/src/transformers/models/gptj/configuration_gptj.py
index 1079169ac3..e30cf2479b 100644
--- a/src/transformers/models/gptj/configuration_gptj.py
+++ b/src/transformers/models/gptj/configuration_gptj.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" GPT-J model configuration """
+""" GPT-J model configuration"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging
@@ -28,11 +28,12 @@ GPTJ_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class GPTJConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`GPTJModel`]. It is used to
- instantiate a GPT-J model according to the specified arguments, defining the model architecture. Instantiating a
- configuration with the defaults will yield a similar configuration to that of the GPT-J [gpt-j-6B](https://huggingface.co/EleutherAI/gpt-j-6B) architecture. Configuration objects inherit from
- [`PretrainedConfig`] and can be used to control the model outputs. Read the documentation from
- [`PretrainedConfig`] for more information.
+ This is the configuration class to store the configuration of a [`GPTJModel`]. It is used to instantiate a GPT-J
+ model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
+ defaults will yield a similar configuration to that of the GPT-J
+ [gpt-j-6B](https://huggingface.co/EleutherAI/gpt-j-6B) architecture. Configuration objects inherit from
+ [`PretrainedConfig`] and can be used to control the model outputs. Read the documentation from [`PretrainedConfig`]
+ for more information.
Args:
vocab_size (`int`, *optional*, defaults to 50400):
diff --git a/src/transformers/models/gptj/modeling_flax_gptj.py b/src/transformers/models/gptj/modeling_flax_gptj.py
index 840d7ed53b..90672a1322 100644
--- a/src/transformers/models/gptj/modeling_flax_gptj.py
+++ b/src/transformers/models/gptj/modeling_flax_gptj.py
@@ -42,12 +42,13 @@ _TOKENIZER_FOR_DOC = "GPTJTokenizer"
GPTJ_START_DOCSTRING = r"""
- This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading or saving, resizing the input
- embeddings, pruning heads etc.)
+ This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a Flax Linen [flax.nn.Module](https://flax.readthedocs.io/en/latest/_autosummary/flax.nn.module.html) subclass. Use it as a regular Flax
- Module and refer to the Flax documentation for all matter related to general usage and behavior.
+ This model is also a Flax Linen
+ [flax.nn.Module](https://flax.readthedocs.io/en/latest/_autosummary/flax.nn.module.html) subclass. Use it as a
+ regular Flax Module and refer to the Flax documentation for all matter related to general usage and behavior.
Finally, this model supports inherent JAX features such as:
@@ -59,11 +60,10 @@ GPTJ_START_DOCSTRING = r"""
Parameters:
config ([`GPTJConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the
- model weights.
+ configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the model weights.
dtype (`jax.numpy.dtype`, *optional*, defaults to `jax.numpy.float32`):
- The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on
- GPUs) and `jax.numpy.bfloat16` (on TPUs).
+ The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on GPUs) and
+ `jax.numpy.bfloat16` (on TPUs).
This can be used to enable mixed-precision training or half-precision inference on GPUs or TPUs. If
specified all the computation will be performed with the given `dtype`.
@@ -71,8 +71,8 @@ GPTJ_START_DOCSTRING = r"""
**Note that this only specifies the dtype of the computation and does not influence the dtype of model
parameters.**
- If you wish to change the dtype of the model parameters, see
- [`~FlaxPreTrainedModel.to_fp16`] and [`~FlaxPreTrainedModel.to_bf16`].
+ If you wish to change the dtype of the model parameters, see [`~FlaxPreTrainedModel.to_fp16`] and
+ [`~FlaxPreTrainedModel.to_bf16`].
"""
GPTJ_INPUTS_DOCSTRING = r"""
@@ -80,9 +80,8 @@ GPTJ_INPUTS_DOCSTRING = r"""
input_ids (`numpy.ndarray` of shape `(batch_size, input_ids_length)`):
`input_ids_length` = `sequence_length`. Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`GPTJTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`GPTJTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`numpy.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
@@ -93,7 +92,8 @@ GPTJ_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
position_ids (`numpy.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
past_key_values (`Dict[str, np.ndarray]`, *optional*, returned by `init_cache` or when passing previous `past_key_values`):
Dictionary of pre-computed hidden-states (key and values in the attention blocks) that can be used for fast
auto-regressive decoding. Pre-computed key and value hidden-states are of shape *[batch_size, max_length]*.
diff --git a/src/transformers/models/gptj/modeling_gptj.py b/src/transformers/models/gptj/modeling_gptj.py
index 05e7f16700..7eafeab20e 100755
--- a/src/transformers/models/gptj/modeling_gptj.py
+++ b/src/transformers/models/gptj/modeling_gptj.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" PyTorch GPT-J model. """
+""" PyTorch GPT-J model."""
from typing import Tuple
@@ -342,8 +342,7 @@ GPTJ_START_DOCSTRING = r"""
Parameters:
config ([`GPTJConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
GPTJ_INPUTS_DOCSTRING = r"""
@@ -351,9 +350,8 @@ GPTJ_INPUTS_DOCSTRING = r"""
input_ids (`torch.LongTensor` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`GPTJTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`GPTJTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
@@ -364,14 +362,16 @@ GPTJ_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.n_positions - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.n_positions - 1]`.
[What are position IDs?](../glossary#position-ids)
head_mask (`torch.FloatTensor` of shape `(num_attention_heads,)` or `(n_layer, num_attention_heads)`, *optional*):
@@ -381,9 +381,9 @@ GPTJ_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_dim)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert *input_ids* indices into associated vectors
- than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -778,8 +778,8 @@ class GPTJForCausalLM(GPTJPreTrainedModel):
r"""
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
- `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to
- `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
+ `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
+ are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -834,9 +834,9 @@ class GPTJForCausalLM(GPTJPreTrainedModel):
@staticmethod
def _reorder_cache(past: Tuple[Tuple[torch.Tensor]], beam_idx: torch.Tensor) -> Tuple[Tuple[torch.Tensor]]:
"""
- This function is used to re-order the `past_key_values` cache if
- [`~PretrainedModel.beam_search`] or [`~PretrainedModel.beam_sample`] is
- called. This is required to match `past_key_values` with the correct beam_idx at every generation step.
+ This function is used to re-order the `past_key_values` cache if [`~PretrainedModel.beam_search`] or
+ [`~PretrainedModel.beam_sample`] is called. This is required to match `past_key_values` with the correct
+ beam_idx at every generation step.
"""
return tuple(
tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past)
@@ -848,14 +848,14 @@ class GPTJForCausalLM(GPTJPreTrainedModel):
"""
The GPT-J Model transformer with a sequence classification head on top (linear layer).
- [`GPTJForSequenceClassification`] uses the last token in order to do the classification, as
- other causal models (e.g. GPT, GPT-2, GPT-Neo) do.
+ [`GPTJForSequenceClassification`] uses the last token in order to do the classification, as other causal models
+ (e.g. GPT, GPT-2, GPT-Neo) do.
Since it does classification on the last token, it requires to know the position of the last token. If a
- `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each
- row. If no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot
- guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take
- the last value in each row of the batch).
+ `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
+ no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
+ padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
+ each row of the batch).
""",
GPTJ_START_DOCSTRING,
)
@@ -899,8 +899,9 @@ class GPTJForSequenceClassification(GPTJPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1024,12 +1025,12 @@ class GPTJForQuestionAnswering(GPTJPreTrainedModel):
r"""
start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the start of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the end of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
diff --git a/src/transformers/models/herbert/tokenization_herbert.py b/src/transformers/models/herbert/tokenization_herbert.py
index c025c02e9b..bd301ed7fe 100644
--- a/src/transformers/models/herbert/tokenization_herbert.py
+++ b/src/transformers/models/herbert/tokenization_herbert.py
@@ -49,8 +49,8 @@ class HerbertTokenizer(XLMTokenizer):
- Such pretokenized input is BPE subtokenized
- This tokenizer inherits from [`XLMTokenizer`] which contains most of the methods. Users should
- refer to the superclass for more information regarding methods.
+ This tokenizer inherits from [`XLMTokenizer`] which contains most of the methods. Users should refer to the
+ superclass for more information regarding methods.
"""
vocab_files_names = VOCAB_FILES_NAMES
diff --git a/src/transformers/models/herbert/tokenization_herbert_fast.py b/src/transformers/models/herbert/tokenization_herbert_fast.py
index 7d08b18983..234ad4a567 100644
--- a/src/transformers/models/herbert/tokenization_herbert_fast.py
+++ b/src/transformers/models/herbert/tokenization_herbert_fast.py
@@ -46,8 +46,8 @@ class HerbertTokenizerFast(PreTrainedTokenizerFast):
- uses BERT's pre-tokenizer: BertPreTokenizer splits tokens on spaces, and also on punctuation. Each occurrence of
a punctuation character will be treated separately.
- This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the methods. Users
- should refer to the superclass for more information regarding methods.
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the methods. Users should refer to the
+ superclass for more information regarding methods.
Args:
vocab_file (`str`):
@@ -160,8 +160,7 @@ class HerbertTokenizerFast(PreTrainedTokenizerFast):
Optional second list of IDs for sequence pairs.
Returns:
- `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given
- sequence(s).
+ `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
"""
sep = [self.sep_token_id]
cls = [self.cls_token_id]
diff --git a/src/transformers/models/hubert/configuration_hubert.py b/src/transformers/models/hubert/configuration_hubert.py
index b1528c03fb..9d481a7555 100644
--- a/src/transformers/models/hubert/configuration_hubert.py
+++ b/src/transformers/models/hubert/configuration_hubert.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" Hubert model configuration """
+""" Hubert model configuration"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging
@@ -28,21 +28,20 @@ HUBERT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class HubertConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`HubertModel`]. It is used to
- instantiate an Hubert model according to the specified arguments, defining the model architecture. Instantiating a
- configuration with the defaults will yield a similar configuration to that of the Hubert
+ This is the configuration class to store the configuration of a [`HubertModel`]. It is used to instantiate an
+ Hubert model according to the specified arguments, defining the model architecture. Instantiating a configuration
+ with the defaults will yield a similar configuration to that of the Hubert
[facebook/hubert-base-ls960](https://huggingface.co/facebook/hubert-base-ls960) architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
vocab_size (`int`, *optional*, defaults to 32):
Vocabulary size of the Hubert model. Defines the number of different tokens that can be represented by the
- `inputs_ids` passed when calling [`HubertModel`]. Vocabulary size of the model.
- Defines the different tokens that can be represented by the *inputs_ids* passed to the forward method of
- [`HubertModel`].
+ `inputs_ids` passed when calling [`HubertModel`]. Vocabulary size of the model. Defines the different
+ tokens that can be represented by the *inputs_ids* passed to the forward method of [`HubertModel`].
hidden_size (`int`, *optional*, defaults to 768):
Dimensionality of the encoder layers and the pooler layer.
num_hidden_layers (`int`, *optional*, defaults to 12):
@@ -52,8 +51,8 @@ class HubertConfig(PretrainedConfig):
intermediate_size (`int`, *optional*, defaults to 3072):
Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"selu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"selu"` and `"gelu_new"` are supported.
hidden_dropout(`float`, *optional*, defaults to 0.1):
The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
attention_dropout(`float`, *optional*, defaults to 0.1):
@@ -93,28 +92,32 @@ class HubertConfig(PretrainedConfig):
num_conv_pos_embedding_groups (`int`, *optional*, defaults to 16):
Number of groups of 1D convolutional positional embeddings layer.
do_stable_layer_norm (`bool`, *optional*, defaults to `False`):
- Whether do apply *stable* layer norm architecture of the Transformer encoder. `do_stable_layer_norm is True` corresponds to applying layer norm before the attention layer, whereas `do_stable_layer_norm is False` corresponds to applying layer norm after the attention layer.
+ Whether do apply *stable* layer norm architecture of the Transformer encoder. `do_stable_layer_norm is
+ True` corresponds to applying layer norm before the attention layer, whereas `do_stable_layer_norm is
+ False` corresponds to applying layer norm after the attention layer.
apply_spec_augment (`bool`, *optional*, defaults to `True`):
Whether to apply *SpecAugment* data augmentation to the outputs of the feature extractor. For reference see
- [SpecAugment: A Simple Data Augmentation Method for Automatic Speech Recognition](https://arxiv.org/abs/1904.08779).
+ [SpecAugment: A Simple Data Augmentation Method for Automatic Speech
+ Recognition](https://arxiv.org/abs/1904.08779).
mask_time_prob (`float`, *optional*, defaults to 0.05):
Percentage (between 0 and 1) of all feature vectors along the time axis which will be masked. The masking
procecure generates ''mask_time_prob*len(time_axis)/mask_time_length'' independent masks over the axis. If
reasoning from the propability of each feature vector to be chosen as the start of the vector span to be
- masked, *mask_time_prob* should be `prob_vector_start*mask_time_length`. Note that overlap may decrease
- the actual percentage of masked vectors. This is only relevant if `apply_spec_augment is True`.
+ masked, *mask_time_prob* should be `prob_vector_start*mask_time_length`. Note that overlap may decrease the
+ actual percentage of masked vectors. This is only relevant if `apply_spec_augment is True`.
mask_time_length (`int`, *optional*, defaults to 10):
Length of vector span along the time axis.
mask_time_min_masks (`int`, *optional*, defaults to 2),:
- The minimum number of masks of length `mask_feature_length` generated along the time axis, each time
- step, irrespectively of `mask_feature_prob`. Only relevant if
- ''mask_time_prob*len(time_axis)/mask_time_length < mask_time_min_masks''
+ The minimum number of masks of length `mask_feature_length` generated along the time axis, each time step,
+ irrespectively of `mask_feature_prob`. Only relevant if ''mask_time_prob*len(time_axis)/mask_time_length <
+ mask_time_min_masks''
mask_feature_prob (`float`, *optional*, defaults to 0.0):
Percentage (between 0 and 1) of all feature vectors along the feature axis which will be masked. The
masking procecure generates ''mask_feature_prob*len(feature_axis)/mask_time_length'' independent masks over
the axis. If reasoning from the propability of each feature vector to be chosen as the start of the vector
- span to be masked, *mask_feature_prob* should be `prob_vector_start*mask_feature_length`. Note that
- overlap may decrease the actual percentage of masked vectors. This is only relevant if `apply_spec_augment is True`.
+ span to be masked, *mask_feature_prob* should be `prob_vector_start*mask_feature_length`. Note that overlap
+ may decrease the actual percentage of masked vectors. This is only relevant if `apply_spec_augment is
+ True`.
mask_feature_length (`int`, *optional*, defaults to 10):
Length of vector span along the feature axis.
mask_feature_min_masks (`int`, *optional*, defaults to 0),:
@@ -125,9 +128,9 @@ class HubertConfig(PretrainedConfig):
Specifies the reduction to apply to the output of `torch.nn.CTCLoss`. Only relevant when training an
instance of [`HubertForCTC`].
ctc_zero_infinity (`bool`, *optional*, defaults to `False`):
- Whether to zero infinite losses and the associated gradients of `torch.nn.CTCLoss`. Infinite losses
- mainly occur when the inputs are too short to be aligned to the targets. Only relevant when training an
- instance of [`HubertForCTC`].
+ Whether to zero infinite losses and the associated gradients of `torch.nn.CTCLoss`. Infinite losses mainly
+ occur when the inputs are too short to be aligned to the targets. Only relevant when training an instance
+ of [`HubertForCTC`].
use_weighted_layer_sum (`bool`, *optional*, defaults to `False`):
Whether to use a weighted average of layer outputs with learned weights. Only relevant when using an
instance of [`HubertForSequenceClassification`].
diff --git a/src/transformers/models/hubert/modeling_hubert.py b/src/transformers/models/hubert/modeling_hubert.py
index 444149c696..f983875c74 100755
--- a/src/transformers/models/hubert/modeling_hubert.py
+++ b/src/transformers/models/hubert/modeling_hubert.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" PyTorch Hubert model. """
+""" PyTorch Hubert model."""
from typing import Optional, Tuple, Union
@@ -65,8 +65,8 @@ def _compute_mask_indices(
) -> np.ndarray:
"""
Computes random mask spans for a given shape. Used to implement [SpecAugment: A Simple Data Augmentation Method for
- ASR](https://arxiv.org/abs/1904.08779). Note that this method is not optimized to run on TPU and should be run
- on CPU as part of the preprocessing during training.
+ ASR](https://arxiv.org/abs/1904.08779). Note that this method is not optimized to run on TPU and should be run on
+ CPU as part of the preprocessing during training.
Args:
shape: The shape for which to compute masks. This should be of a tuple of size 2 where
@@ -838,11 +838,12 @@ class HubertPreTrainedModel(PreTrainedModel):
HUBERT_START_DOCSTRING = r"""
- Hubert was proposed in [HuBERT: Self-Supervised Speech Representation Learning by Masked Prediction of Hidden Units](https://arxiv.org/abs/2106.07447) by Wei-Ning Hsu, Benjamin Bolte, Yao-Hung Hubert Tsai, Kushal Lakhotia,
+ Hubert was proposed in [HuBERT: Self-Supervised Speech Representation Learning by Masked Prediction of Hidden
+ Units](https://arxiv.org/abs/2106.07447) by Wei-Ning Hsu, Benjamin Bolte, Yao-Hung Hubert Tsai, Kushal Lakhotia,
Ruslan Salakhutdinov, Abdelrahman Mohamed.
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving etc.).
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving etc.).
This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use
it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
@@ -851,8 +852,7 @@ HUBERT_START_DOCSTRING = r"""
Parameters:
config ([`HubertConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
@@ -861,11 +861,11 @@ HUBERT_INPUTS_DOCSTRING = r"""
input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`):
Float values of input raw speech waveform. Values can be obtained by loading a *.flac* or *.wav* audio file
into an array of type *List[float]* or a *numpy.ndarray*, *e.g.* via the soundfile library (*pip install
- soundfile*). To prepare the array into *input_values*, the [`Wav2Vec2Processor`] should
- be used for padding and conversion into a tensor of type *torch.FloatTensor*. See
- [`Wav2Vec2Processor.__call__`] for details.
+ soundfile*). To prepare the array into *input_values*, the [`Wav2Vec2Processor`] should be used for padding
+ and conversion into a tensor of type *torch.FloatTensor*. See [`Wav2Vec2Processor.__call__`] for details.
attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0, 1]`:
+ Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0,
+ 1]`:
- 1 for tokens that are **not masked**,
- 0 for tokens that are **masked**.
@@ -874,12 +874,12 @@ HUBERT_INPUTS_DOCSTRING = r"""
- `attention_mask` should only be passed if the corresponding processor has
- `config.return_attention_mask == True`. For all models whose processor has
- `config.return_attention_mask == False`, such as [hubert-base](https://huggingface.co/facebook/hubert-base-ls960), `attention_mask` should **not** be passed
- to avoid degraded performance when doing batched inference. For such models `input_values` should
- simply be padded with 0 and passed without `attention_mask`. Be aware that these models also yield
- slightly different results depending on whether `input_values` is padded or not.
+ `attention_mask` should only be passed if the corresponding processor has `config.return_attention_mask ==
+ True`. For all models whose processor has `config.return_attention_mask == False`, such as
+ [hubert-base](https://huggingface.co/facebook/hubert-base-ls960), `attention_mask` should **not** be passed
+ to avoid degraded performance when doing batched inference. For such models `input_values` should simply be
+ padded with 0 and passed without `attention_mask`. Be aware that these models also yield slightly different
+ results depending on whether `input_values` is padded or not.
@@ -1035,7 +1035,7 @@ class HubertModel(HubertPreTrainedModel):
@add_start_docstrings(
- """Hubert Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC). """,
+ """Hubert Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC).""",
HUBERT_START_DOCSTRING,
)
# Copied from transformers.models.wav2vec2.modeling_wav2vec2.Wav2Vec2ForCTC with Wav2Vec2->Hubert, wav2vec2->hubert, WAV_2_VEC_2->HUBERT
@@ -1084,7 +1084,9 @@ class HubertForCTC(HubertPreTrainedModel):
r"""
labels (`torch.LongTensor` of shape `(batch_size, target_length)`, *optional*):
Labels for connectionist temporal classification. Note that `target_length` has to be smaller or equal to
- the sequence length of the output logits. Indices are selected in `[-100, 0, ..., config.vocab_size - 1]`. All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size - 1]`.
+ the sequence length of the output logits. Indices are selected in `[-100, 0, ..., config.vocab_size - 1]`.
+ All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ...,
+ config.vocab_size - 1]`.
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1199,8 +1201,9 @@ class HubertForSequenceClassification(HubertPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
diff --git a/src/transformers/models/hubert/modeling_tf_hubert.py b/src/transformers/models/hubert/modeling_tf_hubert.py
index b3065af59b..976135dd7e 100644
--- a/src/transformers/models/hubert/modeling_tf_hubert.py
+++ b/src/transformers/models/hubert/modeling_tf_hubert.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" TensorFlow Hubert model. """
+""" TensorFlow Hubert model."""
import inspect
import warnings
from typing import Any, Dict, Optional, Tuple, Union
@@ -56,7 +56,8 @@ LARGE_NEGATIVE = -1e8
def input_values_processing(func, config, input_values, **kwargs):
"""
Process the input of each TensorFlow model including the booleans. In case of a list of symbolic inputs, each input
- has to be named accordingly to the parameters name, i.e. `input_values = tf.keras.Input(shape=(128,), dtype='float32', name="input_values")` otherwise the order of the tensors will not be guaranteed during the
+ has to be named accordingly to the parameters name, i.e. `input_values = tf.keras.Input(shape=(128,),
+ dtype='float32', name="input_values")` otherwise the order of the tensors will not be guaranteed during the
training.
Args:
@@ -206,7 +207,8 @@ def _compute_mask_indices(
shape: the the shape for which to compute masks.
should be of size 2 where first element is batch size and 2nd is timesteps
attention_mask: optional padding mask of the same size as shape, which will prevent masking padded elements
- mask_prob: probability for each token to be chosen as start of the span to be masked. this will be multiplied by
+ mask_prob:
+ probability for each token to be chosen as start of the span to be masked. this will be multiplied by
number of timesteps divided by length of mask span to mask approximately this percentage of all elements.
however due to overlaps, the actual number will be smaller (unless no_overlap is True)
mask_length: size of the mask
@@ -1285,13 +1287,13 @@ class TFHubertPreTrainedModel(TFPreTrainedModel):
HUBERT_START_DOCSTRING = r"""
- This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading or saving, resizing the input
- embeddings, pruning heads etc.)
+ This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use
- it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage
- and behavior.
+ This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it
+ as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and
+ behavior.
@@ -1300,11 +1302,11 @@ HUBERT_START_DOCSTRING = r"""
- having all inputs as keyword arguments (like PyTorch models), or
- having all inputs as a list, tuple or dict in the first positional arguments.
- This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all
- the tensors in the first argument of the model call function: `model(inputs)`.
+ This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all the
+ tensors in the first argument of the model call function: `model(inputs)`.
- If you choose this second option, there are three possibilities you can use to gather all the input Tensors in
- the first positional argument :
+ If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the
+ first positional argument :
- a single Tensor with `input_values` only and nothing else: `model(inputs_ids)`
- a list of varying length with one or several input Tensors IN THE ORDER given in the docstring:
@@ -1317,8 +1319,7 @@ HUBERT_START_DOCSTRING = r"""
Args:
config ([`HubertConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
HUBERT_INPUTS_DOCSTRING = r"""
@@ -1326,9 +1327,8 @@ HUBERT_INPUTS_DOCSTRING = r"""
input_values (`np.ndarray`, `tf.Tensor`, `List[tf.Tensor]` ``Dict[str, tf.Tensor]` or `Dict[str, np.ndarray]` and each example must have the shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`BertTokenizer`]. See
- [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for
- details.
+ Indices can be obtained using [`BertTokenizer`]. See [`PreTrainedTokenizer.__call__`] and
+ [`PreTrainedTokenizer.encode`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`np.ndarray` or `tf.Tensor` of shape `({0})`, *optional*):
@@ -1339,14 +1339,16 @@ HUBERT_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`np.ndarray` or `tf.Tensor` of shape `({0})`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`np.ndarray` or `tf.Tensor` of shape `({0})`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
head_mask (`np.ndarray` or `tf.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
@@ -1356,9 +1358,9 @@ HUBERT_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
inputs_embeds (`np.ndarray` or `tf.Tensor` of shape `({0}, hidden_size)`, *optional*):
- Optionally, instead of passing `input_values` you can choose to directly pass an embedded
- representation. This is useful if you want more control over how to convert `input_values` indices
- into associated vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_values` you can choose to directly pass an embedded representation.
+ This is useful if you want more control over how to convert `input_values` indices into associated vectors
+ than the model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the
@@ -1368,8 +1370,8 @@ HUBERT_INPUTS_DOCSTRING = r"""
more detail. This argument can be used only in eager mode, in graph mode the value in the config will be
used instead.
return_dict (`bool`, *optional*):
- Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This
- argument can be used in eager mode, in graph mode the value will always be set to True.
+ Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This argument can be used
+ in eager mode, in graph mode the value will always be set to True.
training (`bool`, *optional*, defaults to `False``):
Whether or not to use the model in training mode (some modules like dropout modules have different
behaviors between training and evaluation).
@@ -1473,7 +1475,7 @@ class TFHubertModel(TFHubertPreTrainedModel):
@add_start_docstrings(
- """TFHubert Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC). """,
+ """TFHubert Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC).""",
HUBERT_START_DOCSTRING,
)
class TFHubertForCTC(TFHubertPreTrainedModel):
@@ -1509,8 +1511,9 @@ class TFHubertForCTC(TFHubertPreTrainedModel):
) -> Union[TFCausalLMOutput, Tuple[tf.Tensor]]:
r"""
labels (`tf.Tensor` or `np.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_values` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_values` docstring) Tokens with indices set to `-100` are ignored (masked),
+ the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
Returns:
diff --git a/src/transformers/models/ibert/configuration_ibert.py b/src/transformers/models/ibert/configuration_ibert.py
index b389a30bf0..ad0fd8f927 100644
--- a/src/transformers/models/ibert/configuration_ibert.py
+++ b/src/transformers/models/ibert/configuration_ibert.py
@@ -14,7 +14,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" I-BERT configuration """
+""" I-BERT configuration"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging
@@ -31,11 +31,11 @@ IBERT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class IBertConfig(PretrainedConfig):
"""
- This is the configuration class to store the configuration of a [`IBertModel`]. It is used to
- instantiate a I-BERT model according to the specified arguments,
+ This is the configuration class to store the configuration of a [`IBertModel`]. It is used to instantiate a I-BERT
+ model according to the specified arguments,
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
vocab_size (`int`, *optional*, defaults to 30522):
@@ -50,8 +50,8 @@ class IBertConfig(PretrainedConfig):
intermediate_size (`int`, *optional*, defaults to 3072):
Dimensionality of the "intermediate" (often named feed-forward) layer in the Transformer encoder.
hidden_act (`str` or `Callable`, *optional*, defaults to `"gelu"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"silu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"silu"` and `"gelu_new"` are supported.
hidden_dropout_prob (`float`, *optional*, defaults to 0.1):
The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
attention_probs_dropout_prob (`float`, *optional*, defaults to 0.1):
@@ -66,18 +66,19 @@ class IBertConfig(PretrainedConfig):
layer_norm_eps (`float`, *optional*, defaults to 1e-12):
The epsilon used by the layer normalization layers.
position_embedding_type (`str`, *optional*, defaults to `"absolute"`):
- Type of position embedding. Choose one of `"absolute"`, `"relative_key"`,
- `"relative_key_query"`. For positional embeddings use `"absolute"`. For more information on
- `"relative_key"`, please refer to [Self-Attention with Relative Position Representations (Shaw et al.)](https://arxiv.org/abs/1803.02155). For more information on `"relative_key_query"`, please refer to
- *Method 4* in [Improve Transformer Models with Better Relative Position Embeddings (Huang et al.)](https://arxiv.org/abs/2009.13658).
+ Type of position embedding. Choose one of `"absolute"`, `"relative_key"`, `"relative_key_query"`. For
+ positional embeddings use `"absolute"`. For more information on `"relative_key"`, please refer to
+ [Self-Attention with Relative Position Representations (Shaw et al.)](https://arxiv.org/abs/1803.02155).
+ For more information on `"relative_key_query"`, please refer to *Method 4* in [Improve Transformer Models
+ with Better Relative Position Embeddings (Huang et al.)](https://arxiv.org/abs/2009.13658).
quant_mode (`bool`, *optional*, defaults to `False`):
Whether to quantize the model or not.
force_dequant (`str`, *optional*, defaults to `"none"`):
Force dequantize specific nonlinear layer. Dequatized layers are then executed with full precision.
- `"none"`, `"gelu"`, `"softmax"`, `"layernorm"` and `"nonlinear"` are supported. As
- deafult, it is set as `"none"`, which does not dequantize any layers. Please specify `"gelu"`,
- `"softmax"`, or `"layernorm"` to dequantize GELU, Softmax, or LayerNorm, respectively.
- `"nonlinear"` will dequantize all nonlinear layers, i.e., GELU, Softmax, and LayerNorm.
+ `"none"`, `"gelu"`, `"softmax"`, `"layernorm"` and `"nonlinear"` are supported. As deafult, it is set as
+ `"none"`, which does not dequantize any layers. Please specify `"gelu"`, `"softmax"`, or `"layernorm"` to
+ dequantize GELU, Softmax, or LayerNorm, respectively. `"nonlinear"` will dequantize all nonlinear layers,
+ i.e., GELU, Softmax, and LayerNorm.
"""
model_type = "ibert"
diff --git a/src/transformers/models/ibert/modeling_ibert.py b/src/transformers/models/ibert/modeling_ibert.py
index 30b70a669d..b5f2b3f420 100644
--- a/src/transformers/models/ibert/modeling_ibert.py
+++ b/src/transformers/models/ibert/modeling_ibert.py
@@ -15,7 +15,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-"""PyTorch I-BERT model. """
+"""PyTorch I-BERT model."""
import math
@@ -662,19 +662,18 @@ class IBertPreTrainedModel(PreTrainedModel):
IBERT_START_DOCSTRING = r"""
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
Parameters:
config ([`IBertConfig`]): Model configuration class with all the parameters of the
model. Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
IBERT_INPUTS_DOCSTRING = r"""
@@ -682,9 +681,8 @@ IBERT_INPUTS_DOCSTRING = r"""
input_ids (`torch.LongTensor` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`RobertaTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`RobertaTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
@@ -695,14 +693,16 @@ IBERT_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
@@ -712,9 +712,9 @@ IBERT_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -853,7 +853,7 @@ class IBertModel(IBertPreTrainedModel):
)
-@add_start_docstrings("""I-BERT Model with a `language modeling` head on top. """, IBERT_START_DOCSTRING)
+@add_start_docstrings("""I-BERT Model with a `language modeling` head on top.""", IBERT_START_DOCSTRING)
class IBertForMaskedLM(IBertPreTrainedModel):
_keys_to_ignore_on_load_missing = [r"position_ids", r"lm_head.decoder.bias"]
_keys_to_ignore_on_load_unexpected = [r"pooler"]
@@ -896,8 +896,9 @@ class IBertForMaskedLM(IBertPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
kwargs (`Dict[str, any]`, optional, defaults to *{}*):
Used to hide legacy arguments that have been deprecated.
"""
@@ -1003,8 +1004,9 @@ class IBertForSequenceClassification(IBertPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1098,7 +1100,8 @@ class IBertForMultipleChoice(IBertPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
+ Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
+ num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
`input_ids` above)
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1303,12 +1306,12 @@ class IBertForQuestionAnswering(IBertPreTrainedModel):
r"""
start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the start of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the end of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
diff --git a/src/transformers/models/ibert/quant_modules.py b/src/transformers/models/ibert/quant_modules.py
index 83b173a873..d672e70542 100644
--- a/src/transformers/models/ibert/quant_modules.py
+++ b/src/transformers/models/ibert/quant_modules.py
@@ -30,8 +30,7 @@ logger = logging.get_logger(__name__)
class QuantEmbedding(nn.Module):
"""
- Quantized version of `torch.nn.Embedding`. Adds quantization-specific arguments on top of
- `torch.nn.Embedding`.
+ Quantized version of `torch.nn.Embedding`. Adds quantization-specific arguments on top of `torch.nn.Embedding`.
Args:
weight_bit (`int`, *optional*, defaults to `8`):
@@ -358,8 +357,7 @@ class IntGELU(nn.Module):
class IntSoftmax(nn.Module):
"""
- Quantized version of `torch.nn.Softmax`. Adds quantization-specific arguments on top of
- `torch.nn.Softmax`.
+ Quantized version of `torch.nn.Softmax`. Adds quantization-specific arguments on top of `torch.nn.Softmax`.
Args:
output_bit (`int`):
@@ -430,8 +428,7 @@ class IntSoftmax(nn.Module):
class IntLayerNorm(nn.Module):
"""
- Quantized version of `torch.nn.LayerNorm`. Adds quantization-specific arguments on top of
- `torch.nn.LayerNorm`.
+ Quantized version of `torch.nn.LayerNorm`. Adds quantization-specific arguments on top of `torch.nn.LayerNorm`.
Args:
output_bit (`int`, *optional*, defaults to `8`):
@@ -760,8 +757,8 @@ class FixedPointMul(Function):
Scaling factor of the identity tensor *identity*, if exists.
Returns:
- `torch.Tensor`: Output tensor(*pre_act* if *identity* is not given, otherwise the addition of *pre_act*
- and *identity*), whose scale is rescaled to *z_scaling_factor*.
+ `torch.Tensor`: Output tensor(*pre_act* if *identity* is not given, otherwise the addition of *pre_act* and
+ *identity*), whose scale is rescaled to *z_scaling_factor*.
"""
@staticmethod
diff --git a/src/transformers/models/imagegpt/configuration_imagegpt.py b/src/transformers/models/imagegpt/configuration_imagegpt.py
index 5cfec7e4b3..d52414abfd 100644
--- a/src/transformers/models/imagegpt/configuration_imagegpt.py
+++ b/src/transformers/models/imagegpt/configuration_imagegpt.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" OpenAI ImageGPT configuration """
+""" OpenAI ImageGPT configuration"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging
@@ -29,20 +29,19 @@ IMAGEGPT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class ImageGPTConfig(PretrainedConfig):
"""
- This is the configuration class to store the configuration of a [`ImageGPTModel`] or a
- [`TFImageGPTModel`]. It is used to instantiate a GPT-2 model according to the specified
- arguments, defining the model architecture. Instantiating a configuration with the defaults will yield a similar
- configuration to that of the ImageGPT [small](https://huggingface.co/imagegpt) architecture.
+ This is the configuration class to store the configuration of a [`ImageGPTModel`] or a [`TFImageGPTModel`]. It is
+ used to instantiate a GPT-2 model according to the specified arguments, defining the model architecture.
+ Instantiating a configuration with the defaults will yield a similar configuration to that of the ImageGPT
+ [small](https://huggingface.co/imagegpt) architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
vocab_size (`int`, *optional*, defaults to 512):
Vocabulary size of the GPT-2 model. Defines the number of different tokens that can be represented by the
- `inputs_ids` passed when calling [`ImageGPTModel`] or
- [`TFImageGPTModel`].
+ `inputs_ids` passed when calling [`ImageGPTModel`] or [`TFImageGPTModel`].
n_positions (`int`, *optional*, defaults to 32*32):
The maximum sequence length that this model might ever be used with. Typically set this to something large
just in case (e.g., 512 or 1024 or 2048).
diff --git a/src/transformers/models/imagegpt/feature_extraction_imagegpt.py b/src/transformers/models/imagegpt/feature_extraction_imagegpt.py
index a6a069afff..223587f1c7 100644
--- a/src/transformers/models/imagegpt/feature_extraction_imagegpt.py
+++ b/src/transformers/models/imagegpt/feature_extraction_imagegpt.py
@@ -49,8 +49,8 @@ class ImageGPTFeatureExtractor(FeatureExtractionMixin, ImageFeatureExtractionMix
resolution (such as 32x32 or 64x64), normalize them and finally color quantize them to obtain sequences of "pixel
values" (color clusters).
- This feature extractor inherits from [`FeatureExtractionMixin`] which contains most of the main
- methods. Users should refer to this superclass for more information regarding those methods.
+ This feature extractor inherits from [`FeatureExtractionMixin`] which contains most of the main methods. Users
+ should refer to this superclass for more information regarding those methods.
Args:
clusters (`np.ndarray`):
@@ -59,12 +59,12 @@ class ImageGPTFeatureExtractor(FeatureExtractionMixin, ImageFeatureExtractionMix
Whether to resize the input to a certain `size`.
size (`int` or `Tuple(int)`, *optional*, defaults to 32):
Resize the input to the given size. If a tuple is provided, it should be (width, height). If only an
- integer is provided, then the input will be resized to (size, size). Only has an effect if `do_resize`
- is set to `True`.
+ integer is provided, then the input will be resized to (size, size). Only has an effect if `do_resize` is
+ set to `True`.
resample (`int`, *optional*, defaults to `PIL.Image.BILINEAR`):
An optional resampling filter. This can be one of `PIL.Image.NEAREST`, `PIL.Image.BOX`,
- `PIL.Image.BILINEAR`, `PIL.Image.HAMMING`, `PIL.Image.BICUBIC` or `PIL.Image.LANCZOS`.
- Only has an effect if `do_resize` is set to `True`.
+ `PIL.Image.BILINEAR`, `PIL.Image.HAMMING`, `PIL.Image.BICUBIC` or `PIL.Image.LANCZOS`. Only has an effect
+ if `do_resize` is set to `True`.
do_normalize (`bool`, *optional*, defaults to `True`):
Whether or not to normalize the input to the range between -1 and +1.
"""
diff --git a/src/transformers/models/imagegpt/modeling_imagegpt.py b/src/transformers/models/imagegpt/modeling_imagegpt.py
index e1568ab73a..563e1b0279 100755
--- a/src/transformers/models/imagegpt/modeling_imagegpt.py
+++ b/src/transformers/models/imagegpt/modeling_imagegpt.py
@@ -535,19 +535,18 @@ class ImageGPTPreTrainedModel(PreTrainedModel):
IMAGEGPT_START_DOCSTRING = r"""
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
Parameters:
config ([`ImageGPTConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
IMAGEGPT_INPUTS_DOCSTRING = r"""
@@ -557,17 +556,16 @@ IMAGEGPT_INPUTS_DOCSTRING = r"""
`past_key_values[0][0].shape[-2]` (`sequence_length` of input past key value states). Indices of input
sequence tokens in the vocabulary.
- If `past_key_values` is used, only `input_ids` that do not have their past calculated should be
- passed as `input_ids`.
+ If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as
+ `input_ids`.
- Indices can be obtained using [`ImageGPTFeatureExtractor`]. See
- [`ImageGPTFeatureExtractor.__call__`] for details.
+ Indices can be obtained using [`ImageGPTFeatureExtractor`]. See [`ImageGPTFeatureExtractor.__call__`] for
+ details.
past_key_values (`Tuple[Tuple[torch.Tensor]]` of length `config.n_layers`):
Contains precomputed hidden-states (key and values in the attention blocks) as computed by the model (see
- `past_key_values` output below). Can be used to speed up sequential decoding. The `input_ids` which
- have their past given to this model should not be passed as `input_ids` as they have already been
- computed.
+ `past_key_values` output below). Can be used to speed up sequential decoding. The `input_ids` which have
+ their past given to this model should not be passed as `input_ids` as they have already been computed.
attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
@@ -576,14 +574,16 @@ IMAGEGPT_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
@@ -593,15 +593,15 @@ IMAGEGPT_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
If `past_key_values` is used, optionally only the last `inputs_embeds` have to be input (see
`past_key_values`).
use_cache (`bool`, *optional*):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`).
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`).
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -674,8 +674,8 @@ class ImageGPTModel(ImageGPTPreTrainedModel):
r"""
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
- `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to
- `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
+ `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
+ are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
Returns:
@@ -968,8 +968,8 @@ class ImageGPTForCausalImageModeling(ImageGPTPreTrainedModel):
r"""
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
- `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to
- `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
+ `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
+ are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
Returns:
@@ -1064,9 +1064,9 @@ class ImageGPTForCausalImageModeling(ImageGPTPreTrainedModel):
@staticmethod
def _reorder_cache(past: Tuple[Tuple[torch.Tensor]], beam_idx: torch.Tensor) -> Tuple[Tuple[torch.Tensor]]:
"""
- This function is used to re-order the `past_key_values` cache if
- [`~PreTrainedModel.beam_search`] or [`~PreTrainedModel.beam_sample`] is
- called. This is required to match `past_key_values` with the correct beam_idx at every generation step.
+ This function is used to re-order the `past_key_values` cache if [`~PreTrainedModel.beam_search`] or
+ [`~PreTrainedModel.beam_sample`] is called. This is required to match `past_key_values` with the correct
+ beam_idx at every generation step.
"""
return tuple(
tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past)
@@ -1077,8 +1077,7 @@ class ImageGPTForCausalImageModeling(ImageGPTPreTrainedModel):
@add_start_docstrings(
"""
The ImageGPT Model transformer with an image classification head on top (linear layer).
- [`ImageGPTForImageClassification`] average-pools the hidden states in order to do the
- classification.
+ [`ImageGPTForImageClassification`] average-pools the hidden states in order to do the classification.
""",
IMAGEGPT_START_DOCSTRING,
)
@@ -1114,8 +1113,9 @@ class ImageGPTForImageClassification(ImageGPTPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
Returns:
diff --git a/src/transformers/models/layoutlm/configuration_layoutlm.py b/src/transformers/models/layoutlm/configuration_layoutlm.py
index 57b3bb4637..6b7e0537e0 100644
--- a/src/transformers/models/layoutlm/configuration_layoutlm.py
+++ b/src/transformers/models/layoutlm/configuration_layoutlm.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" LayoutLM model configuration """
+""" LayoutLM model configuration"""
from collections import OrderedDict
from typing import Any, List, Mapping, Optional
@@ -34,12 +34,13 @@ LAYOUTLM_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class LayoutLMConfig(BertConfig):
r"""
- This is the configuration class to store the configuration of a [`LayoutLMModel`]. It is used to
- instantiate a LayoutLM model according to the specified arguments, defining the model architecture. Instantiating a
- configuration with the defaults will yield a similar configuration to that of the LayoutLM [layoutlm-base-uncased](https://huggingface.co/microsoft/layoutlm-base-uncased) architecture.
+ This is the configuration class to store the configuration of a [`LayoutLMModel`]. It is used to instantiate a
+ LayoutLM model according to the specified arguments, defining the model architecture. Instantiating a configuration
+ with the defaults will yield a similar configuration to that of the LayoutLM
+ [layoutlm-base-uncased](https://huggingface.co/microsoft/layoutlm-base-uncased) architecture.
- Configuration objects inherit from [`BertConfig`] and can be used to control the model outputs.
- Read the documentation from [`BertConfig`] for more information.
+ Configuration objects inherit from [`BertConfig`] and can be used to control the model outputs. Read the
+ documentation from [`BertConfig`] for more information.
Args:
@@ -55,8 +56,8 @@ class LayoutLMConfig(BertConfig):
intermediate_size (`int`, *optional*, defaults to 3072):
Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"silu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"silu"` and `"gelu_new"` are supported.
hidden_dropout_prob (`float`, *optional*, defaults to 0.1):
The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
attention_probs_dropout_prob (`float`, *optional*, defaults to 0.1):
diff --git a/src/transformers/models/layoutlm/modeling_layoutlm.py b/src/transformers/models/layoutlm/modeling_layoutlm.py
index a5e6df5d59..0230bf9d9c 100644
--- a/src/transformers/models/layoutlm/modeling_layoutlm.py
+++ b/src/transformers/models/layoutlm/modeling_layoutlm.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" PyTorch LayoutLM model. """
+""" PyTorch LayoutLM model."""
import math
@@ -638,7 +638,9 @@ class LayoutLMPreTrainedModel(PreTrainedModel):
LAYOUTLM_START_DOCSTRING = r"""
- The LayoutLM model was proposed in [LayoutLM: Pre-training of Text and Layout for Document Image Understanding](https://arxiv.org/abs/1912.13318) by Yiheng Xu, Minghao Li, Lei Cui, Shaohan Huang, Furu Wei and Ming Zhou.
+ The LayoutLM model was proposed in [LayoutLM: Pre-training of Text and Layout for Document Image
+ Understanding](https://arxiv.org/abs/1912.13318) by Yiheng Xu, Minghao Li, Lei Cui, Shaohan Huang, Furu Wei and
+ Ming Zhou.
This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use
it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
@@ -647,8 +649,7 @@ LAYOUTLM_START_DOCSTRING = r"""
Parameters:
config ([`LayoutLMConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
LAYOUTLM_INPUTS_DOCSTRING = r"""
@@ -656,44 +657,45 @@ LAYOUTLM_INPUTS_DOCSTRING = r"""
input_ids (`torch.LongTensor` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`LayoutLMTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`LayoutLMTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
bbox (`torch.LongTensor` of shape `({0}, 4)`, *optional*):
- Bounding boxes of each input sequence tokens. Selected in the range `[0, config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1,
- y1) format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and
- (x1, y1) represents the position of the lower right corner. See [Overview](#Overview) for normalization.
+ Bounding boxes of each input sequence tokens. Selected in the range `[0,
+ config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1)
+ format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1,
+ y1) represents the position of the lower right corner. See [Overview](#Overview) for normalization.
attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: `1` for
tokens that are NOT MASKED, `0` for MASKED tokens.
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`: `0` corresponds to a *sentence A* token, `1` corresponds to a *sentence B* token
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`: `0` corresponds to a *sentence A* token, `1` corresponds to a *sentence B* token
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: `1`
indicates the head is **not masked**, `0` indicates the head is **masked**.
inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert *input_ids* indices into associated vectors
- than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
If set to `True`, the attentions tensors of all attention layers are returned. See `attentions` under
returned tensors for more detail.
output_hidden_states (`bool`, *optional*):
- If set to `True`, the hidden states of all layers are returned. See `hidden_states` under returned
- tensors for more detail.
+ If set to `True`, the hidden states of all layers are returned. See `hidden_states` under returned tensors
+ for more detail.
return_dict (`bool`, *optional*):
- If set to `True`, the model will return a [`~file_utils.ModelOutput`] instead of a
- plain tuple.
+ If set to `True`, the model will return a [`~file_utils.ModelOutput`] instead of a plain tuple.
"""
@@ -846,7 +848,7 @@ class LayoutLMModel(LayoutLMPreTrainedModel):
)
-@add_start_docstrings("""LayoutLM Model with a `language modeling` head on top. """, LAYOUTLM_START_DOCSTRING)
+@add_start_docstrings("""LayoutLM Model with a `language modeling` head on top.""", LAYOUTLM_START_DOCSTRING)
class LayoutLMForMaskedLM(LayoutLMPreTrainedModel):
def __init__(self, config):
super().__init__(config)
@@ -886,8 +888,9 @@ class LayoutLMForMaskedLM(LayoutLMPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
Returns:
@@ -1002,8 +1005,9 @@ class LayoutLMForSequenceClassification(LayoutLMPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
Returns:
diff --git a/src/transformers/models/layoutlm/modeling_tf_layoutlm.py b/src/transformers/models/layoutlm/modeling_tf_layoutlm.py
index aa33734d4d..90017db10b 100644
--- a/src/transformers/models/layoutlm/modeling_tf_layoutlm.py
+++ b/src/transformers/models/layoutlm/modeling_tf_layoutlm.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" TF 2.0 LayoutLM model. """
+""" TF 2.0 LayoutLM model."""
import math
import warnings
@@ -823,13 +823,13 @@ class TFLayoutLMPreTrainedModel(TFPreTrainedModel):
LAYOUTLM_START_DOCSTRING = r"""
- This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading or saving, resizing the input
- embeddings, pruning heads etc.)
+ This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use
- it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage
- and behavior.
+ This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it
+ as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and
+ behavior.
@@ -838,11 +838,11 @@ LAYOUTLM_START_DOCSTRING = r"""
- having all inputs as keyword arguments (like PyTorch models), or
- having all inputs as a list, tuple or dict in the first positional arguments.
- This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all
- the tensors in the first argument of the model call function: `model(inputs)`.
+ This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all the
+ tensors in the first argument of the model call function: `model(inputs)`.
- If you choose this second option, there are three possibilities you can use to gather all the input Tensors in
- the first positional argument :
+ If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the
+ first positional argument :
- a single Tensor with `input_ids` only and nothing else: `model(inputs_ids)`
- a list of varying length with one or several input Tensors IN THE ORDER given in the docstring:
@@ -855,8 +855,7 @@ LAYOUTLM_START_DOCSTRING = r"""
Args:
config ([`LayoutLMConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~TFPreTrainedModel.from_pretrained`] method to load the
- model weights.
+ configuration. Check out the [`~TFPreTrainedModel.from_pretrained`] method to load the model weights.
"""
LAYOUTLM_INPUTS_DOCSTRING = r"""
@@ -864,13 +863,13 @@ LAYOUTLM_INPUTS_DOCSTRING = r"""
input_ids (`Numpy array` or `tf.Tensor` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`LayoutLMTokenizer`]. See
- [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for
- details.
+ Indices can be obtained using [`LayoutLMTokenizer`]. See [`PreTrainedTokenizer.__call__`] and
+ [`PreTrainedTokenizer.encode`] for details.
[What are input IDs?](../glossary#input-ids)
bbox (`Numpy array` or `tf.Tensor` of shape `({0}, 4)`, *optional*):
- Bounding Boxes of each input sequence tokens. Selected in the range `[0, config.max_2d_position_embeddings- 1]`.
+ Bounding Boxes of each input sequence tokens. Selected in the range `[0, config.max_2d_position_embeddings-
+ 1]`.
attention_mask (`Numpy array` or `tf.Tensor` of shape `({0})`, *optional*):
Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
@@ -879,14 +878,16 @@ LAYOUTLM_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`Numpy array` or `tf.Tensor` of shape `({0})`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`Numpy array` or `tf.Tensor` of shape `({0})`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
head_mask (`Numpy array` or `tf.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
@@ -896,9 +897,9 @@ LAYOUTLM_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
inputs_embeds (`tf.Tensor` of shape `({0}, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -1032,7 +1033,7 @@ class TFLayoutLMModel(TFLayoutLMPreTrainedModel):
)
-@add_start_docstrings("""LayoutLM Model with a `language modeling` head on top. """, LAYOUTLM_START_DOCSTRING)
+@add_start_docstrings("""LayoutLM Model with a `language modeling` head on top.""", LAYOUTLM_START_DOCSTRING)
class TFLayoutLMForMaskedLM(TFLayoutLMPreTrainedModel, TFMaskedLanguageModelingLoss):
# names with a '.' represents the authorized unexpected/missing layers when a TF model is loaded from a PT model
_keys_to_ignore_on_load_unexpected = [
@@ -1081,8 +1082,9 @@ class TFLayoutLMForMaskedLM(TFLayoutLMPreTrainedModel, TFMaskedLanguageModelingL
) -> Union[TFMaskedLMOutput, Tuple[tf.Tensor]]:
r"""
labels (`tf.Tensor` or `np.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
Returns:
@@ -1217,8 +1219,9 @@ class TFLayoutLMForSequenceClassification(TFLayoutLMPreTrainedModel, TFSequenceC
) -> Union[TFSequenceClassifierOutput, Tuple[tf.Tensor]]:
r"""
labels (`tf.Tensor` or `np.ndarray` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
Returns:
diff --git a/src/transformers/models/layoutlm/tokenization_layoutlm.py b/src/transformers/models/layoutlm/tokenization_layoutlm.py
index 603d730e8a..6ef9a9c3a0 100644
--- a/src/transformers/models/layoutlm/tokenization_layoutlm.py
+++ b/src/transformers/models/layoutlm/tokenization_layoutlm.py
@@ -47,11 +47,10 @@ class LayoutLMTokenizer(BertTokenizer):
r"""
Constructs a LayoutLM tokenizer.
- [`LayoutLMTokenizer`] is identical to [`BertTokenizer`] and runs end-to-end
- tokenization: punctuation splitting + wordpiece.
+ [`LayoutLMTokenizer`] is identical to [`BertTokenizer`] and runs end-to-end tokenization: punctuation splitting +
+ wordpiece.
- Refer to superclass [`BertTokenizer`] for usage examples and documentation concerning
- parameters.
+ Refer to superclass [`BertTokenizer`] for usage examples and documentation concerning parameters.
"""
vocab_files_names = VOCAB_FILES_NAMES
diff --git a/src/transformers/models/layoutlm/tokenization_layoutlm_fast.py b/src/transformers/models/layoutlm/tokenization_layoutlm_fast.py
index 8e8e13a90f..90ba0a94fe 100644
--- a/src/transformers/models/layoutlm/tokenization_layoutlm_fast.py
+++ b/src/transformers/models/layoutlm/tokenization_layoutlm_fast.py
@@ -52,11 +52,10 @@ class LayoutLMTokenizerFast(BertTokenizerFast):
r"""
Constructs a "Fast" LayoutLMTokenizer.
- [`LayoutLMTokenizerFast`] is identical to [`BertTokenizerFast`] and runs
- end-to-end tokenization: punctuation splitting + wordpiece.
+ [`LayoutLMTokenizerFast`] is identical to [`BertTokenizerFast`] and runs end-to-end tokenization: punctuation
+ splitting + wordpiece.
- Refer to superclass [`BertTokenizerFast`] for usage examples and documentation concerning
- parameters.
+ Refer to superclass [`BertTokenizerFast`] for usage examples and documentation concerning parameters.
"""
vocab_files_names = VOCAB_FILES_NAMES
diff --git a/src/transformers/models/layoutlmv2/configuration_layoutlmv2.py b/src/transformers/models/layoutlmv2/configuration_layoutlmv2.py
index de19988365..430cb9dd74 100644
--- a/src/transformers/models/layoutlmv2/configuration_layoutlmv2.py
+++ b/src/transformers/models/layoutlmv2/configuration_layoutlmv2.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" LayoutLMv2 model configuration """
+""" LayoutLMv2 model configuration"""
from ...configuration_utils import PretrainedConfig
from ...file_utils import is_detectron2_available
@@ -34,19 +34,18 @@ if is_detectron2_available():
class LayoutLMv2Config(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`LayoutLMv2Model`]. It is used
- to instantiate an LayoutLMv2 model according to the specified arguments, defining the model architecture.
- Instantiating a configuration with the defaults will yield a similar configuration to that of the LayoutLMv2
+ This is the configuration class to store the configuration of a [`LayoutLMv2Model`]. It is used to instantiate an
+ LayoutLMv2 model according to the specified arguments, defining the model architecture. Instantiating a
+ configuration with the defaults will yield a similar configuration to that of the LayoutLMv2
[microsoft/layoutlmv2-base-uncased](https://huggingface.co/microsoft/layoutlmv2-base-uncased) architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
vocab_size (`int`, *optional*, defaults to 30522):
Vocabulary size of the LayoutLMv2 model. Defines the number of different tokens that can be represented by
- the `inputs_ids` passed when calling [`LayoutLMv2Model`] or
- [`TFLayoutLMv2Model`].
+ the `inputs_ids` passed when calling [`LayoutLMv2Model`] or [`TFLayoutLMv2Model`].
hidden_size (`int`, *optional*, defaults to 768):
Dimension of the encoder layers and the pooler layer.
num_hidden_layers (`int`, *optional*, defaults to 12):
@@ -56,8 +55,8 @@ class LayoutLMv2Config(PretrainedConfig):
intermediate_size (`int`, *optional*, defaults to 3072):
Dimension of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"selu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"selu"` and `"gelu_new"` are supported.
hidden_dropout_prob (`float`, *optional*, defaults to 0.1):
The dropout probabilitiy for all fully connected layers in the embeddings, encoder, and pooler.
attention_probs_dropout_prob (`float`, *optional*, defaults to 0.1):
@@ -66,8 +65,8 @@ class LayoutLMv2Config(PretrainedConfig):
The maximum sequence length that this model might ever be used with. Typically set this to something large
just in case (e.g., 512 or 1024 or 2048).
type_vocab_size (`int`, *optional*, defaults to 2):
- The vocabulary size of the `token_type_ids` passed when calling [`LayoutLMv2Model`]
- or [`TFLayoutLMv2Model`].
+ The vocabulary size of the `token_type_ids` passed when calling [`LayoutLMv2Model`] or
+ [`TFLayoutLMv2Model`].
initializer_range (`float`, *optional*, defaults to 0.02):
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
layer_norm_eps (`float`, *optional*, defaults to 1e-12):
@@ -98,7 +97,8 @@ class LayoutLMv2Config(PretrainedConfig):
has_visual_segment_embedding (`bool`, *optional*, defaults to `False`):
Whether or not to add visual segment embeddings.
detectron2_config_args (`dict`, *optional*):
- Dictionary containing the configuration arguments of the Detectron2 visual backbone. Refer to [this file](https://github.com/microsoft/unilm/blob/master/layoutlmft/layoutlmft/models/layoutlmv2/detectron2_config.py)
+ Dictionary containing the configuration arguments of the Detectron2 visual backbone. Refer to [this
+ file](https://github.com/microsoft/unilm/blob/master/layoutlmft/layoutlmft/models/layoutlmv2/detectron2_config.py)
for details regarding default values.
Example:
diff --git a/src/transformers/models/layoutlmv2/feature_extraction_layoutlmv2.py b/src/transformers/models/layoutlmv2/feature_extraction_layoutlmv2.py
index b10cedf4ed..6cc19ccdac 100644
--- a/src/transformers/models/layoutlmv2/feature_extraction_layoutlmv2.py
+++ b/src/transformers/models/layoutlmv2/feature_extraction_layoutlmv2.py
@@ -85,21 +85,20 @@ class LayoutLMv2FeatureExtractor(FeatureExtractionMixin, ImageFeatureExtractionM
Constructs a LayoutLMv2 feature extractor. This can be used to resize document images to the same size, as well as
to apply OCR on them in order to get a list of words and normalized bounding boxes.
- This feature extractor inherits from [`~feature_extraction_utils.PreTrainedFeatureExtractor`]
- which contains most of the main methods. Users should refer to this superclass for more information regarding those
- methods.
+ This feature extractor inherits from [`~feature_extraction_utils.PreTrainedFeatureExtractor`] which contains most
+ of the main methods. Users should refer to this superclass for more information regarding those methods.
Args:
do_resize (`bool`, *optional*, defaults to `True`):
Whether to resize the input to a certain `size`.
size (`int` or `Tuple(int)`, *optional*, defaults to 224):
Resize the input to the given size. If a tuple is provided, it should be (width, height). If only an
- integer is provided, then the input will be resized to (size, size). Only has an effect if `do_resize`
- is set to `True`.
+ integer is provided, then the input will be resized to (size, size). Only has an effect if `do_resize` is
+ set to `True`.
resample (`int`, *optional*, defaults to `PIL.Image.BILINEAR`):
An optional resampling filter. This can be one of `PIL.Image.NEAREST`, `PIL.Image.BOX`,
- `PIL.Image.BILINEAR`, `PIL.Image.HAMMING`, `PIL.Image.BICUBIC` or `PIL.Image.LANCZOS`.
- Only has an effect if `do_resize` is set to `True`.
+ `PIL.Image.BILINEAR`, `PIL.Image.HAMMING`, `PIL.Image.BICUBIC` or `PIL.Image.LANCZOS`. Only has an effect
+ if `do_resize` is set to `True`.
apply_ocr (`bool`, *optional*, defaults to `True`):
Whether to apply the Tesseract OCR engine to get words + normalized bounding boxes.
ocr_lang (`Optional[str]`, *optional*):
@@ -148,11 +147,10 @@ class LayoutLMv2FeatureExtractor(FeatureExtractionMixin, ImageFeatureExtractionM
- **pixel_values** -- Pixel values to be fed to a model, of shape (batch_size, num_channels, height,
width).
- - **words** -- Optional words as identified by Tesseract OCR (only when
- [`LayoutLMv2FeatureExtractor`] was initialized with `apply_ocr` set to `True`).
+ - **words** -- Optional words as identified by Tesseract OCR (only when [`LayoutLMv2FeatureExtractor`] was
+ initialized with `apply_ocr` set to `True`).
- **boxes** -- Optional bounding boxes as identified by Tesseract OCR, normalized based on the image size
- (only when [`LayoutLMv2FeatureExtractor`] was initialized with `apply_ocr` set to
- `True`).
+ (only when [`LayoutLMv2FeatureExtractor`] was initialized with `apply_ocr` set to `True`).
Examples:
diff --git a/src/transformers/models/layoutlmv2/modeling_layoutlmv2.py b/src/transformers/models/layoutlmv2/modeling_layoutlmv2.py
index 00198046aa..0b0cee27cb 100755
--- a/src/transformers/models/layoutlmv2/modeling_layoutlmv2.py
+++ b/src/transformers/models/layoutlmv2/modeling_layoutlmv2.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" PyTorch LayoutLMv2 model. """
+""" PyTorch LayoutLMv2 model."""
import math
@@ -622,8 +622,7 @@ LAYOUTLMV2_START_DOCSTRING = r"""
Parameters:
config ([`LayoutLMv2Config`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
LAYOUTLMV2_INPUTS_DOCSTRING = r"""
@@ -631,16 +630,16 @@ LAYOUTLMV2_INPUTS_DOCSTRING = r"""
input_ids (`torch.LongTensor` of shape `{0}`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`LayoutLMv2Tokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`LayoutLMv2Tokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
bbox (`torch.LongTensor` of shape `({0}, 4)`, *optional*):
- Bounding boxes of each input sequence tokens. Selected in the range `[0, config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1,
- y1) format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and
- (x1, y1) represents the position of the lower right corner.
+ Bounding boxes of each input sequence tokens. Selected in the range `[0,
+ config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1)
+ format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1,
+ y1) represents the position of the lower right corner.
image (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` or `detectron.structures.ImageList` whose `tensors` is of shape `(batch_size, num_channels, height, width)`):
Batch of document images.
@@ -653,14 +652,16 @@ LAYOUTLMV2_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`torch.LongTensor` of shape `{0}`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`torch.LongTensor` of shape `{0}`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
@@ -670,9 +671,9 @@ LAYOUTLMV2_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert *input_ids* indices into associated vectors
- than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -981,8 +982,9 @@ class LayoutLMv2ForSequenceClassification(LayoutLMv2PreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
Returns:
@@ -1227,8 +1229,8 @@ class LayoutLMv2ForTokenClassification(LayoutLMv2PreTrainedModel):
@add_start_docstrings(
"""
LayoutLMv2 Model with a span classification head on top for extractive question-answering tasks such as
- [DocVQA](https://rrc.cvc.uab.es/?ch=17) (a linear layer on top of the text part of the hidden-states output to compute
- `span start logits` and `span end logits`).
+ [DocVQA](https://rrc.cvc.uab.es/?ch=17) (a linear layer on top of the text part of the hidden-states output to
+ compute `span start logits` and `span end logits`).
""",
LAYOUTLMV2_START_DOCSTRING,
)
@@ -1267,12 +1269,12 @@ class LayoutLMv2ForQuestionAnswering(LayoutLMv2PreTrainedModel):
r"""
start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the start of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the end of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
Returns:
diff --git a/src/transformers/models/layoutlmv2/processing_layoutlmv2.py b/src/transformers/models/layoutlmv2/processing_layoutlmv2.py
index d49dbc99bb..58b24a8adb 100644
--- a/src/transformers/models/layoutlmv2/processing_layoutlmv2.py
+++ b/src/transformers/models/layoutlmv2/processing_layoutlmv2.py
@@ -31,20 +31,17 @@ class LayoutLMv2Processor:
[`LayoutLMv2Processor`] offers all the functionalities you need to prepare data for the model.
- It first uses [`LayoutLMv2FeatureExtractor`] to resize document images to a fixed size, and
- optionally applies OCR to get words and normalized bounding boxes. These are then provided to
- [`LayoutLMv2Tokenizer`] or [`LayoutLMv2TokenizerFast`], which turns the words
- and bounding boxes into token-level `input_ids`, `attention_mask`, `token_type_ids`, `bbox`.
- Optionally, one can provide integer `word_labels`, which are turned into token-level `labels` for token
- classification tasks (such as FUNSD, CORD).
+ It first uses [`LayoutLMv2FeatureExtractor`] to resize document images to a fixed size, and optionally applies OCR
+ to get words and normalized bounding boxes. These are then provided to [`LayoutLMv2Tokenizer`] or
+ [`LayoutLMv2TokenizerFast`], which turns the words and bounding boxes into token-level `input_ids`,
+ `attention_mask`, `token_type_ids`, `bbox`. Optionally, one can provide integer `word_labels`, which are turned
+ into token-level `labels` for token classification tasks (such as FUNSD, CORD).
Args:
feature_extractor (`LayoutLMv2FeatureExtractor`):
- An instance of [`LayoutLMv2FeatureExtractor`]. The feature extractor is a required
- input.
+ An instance of [`LayoutLMv2FeatureExtractor`]. The feature extractor is a required input.
tokenizer (`LayoutLMv2Tokenizer` or `LayoutLMv2TokenizerFast`):
- An instance of [`LayoutLMv2Tokenizer`] or
- [`LayoutLMv2TokenizerFast`]. The tokenizer is a required input.
+ An instance of [`LayoutLMv2Tokenizer`] or [`LayoutLMv2TokenizerFast`]. The tokenizer is a required input.
"""
def __init__(self, feature_extractor, tokenizer):
@@ -67,10 +64,9 @@ class LayoutLMv2Processor:
- This class method is simply calling
- [`~feature_extraction_utils.FeatureExtractionMixin.save_pretrained`] and
- [`~tokenization_utils_base.PreTrainedTokenizer.save_pretrained`]. Please refer to the
- docstrings of the methods above for more information.
+ This class method is simply calling [`~feature_extraction_utils.FeatureExtractionMixin.save_pretrained`] and
+ [`~tokenization_utils_base.PreTrainedTokenizer.save_pretrained`]. Please refer to the docstrings of the methods
+ above for more information.
@@ -91,10 +87,9 @@ class LayoutLMv2Processor:
This class method is simply calling LayoutLMv2FeatureExtractor's
- [`~feature_extraction_utils.FeatureExtractionMixin.from_pretrained`] and
- LayoutLMv2TokenizerFast's
- [`~tokenization_utils_base.PreTrainedTokenizer.from_pretrained`]. Please refer to the
- docstrings of the methods above for more information.
+ [`~feature_extraction_utils.FeatureExtractionMixin.from_pretrained`] and LayoutLMv2TokenizerFast's
+ [`~tokenization_utils_base.PreTrainedTokenizer.from_pretrained`]. Please refer to the docstrings of the methods
+ above for more information.
@@ -106,8 +101,7 @@ class LayoutLMv2Processor:
huggingface.co. Valid model ids can be located at the root-level, like `bert-base-uncased`, or
namespaced under a user or organization name, like `dbmdz/bert-base-german-cased`.
- a path to a *directory* containing a feature extractor file saved using the
- [`~SequenceFeatureExtractor.save_pretrained`] method, e.g.,
- `./my_model_directory/`.
+ [`~SequenceFeatureExtractor.save_pretrained`] method, e.g., `./my_model_directory/`.
- a path or url to a saved feature extractor JSON *file*, e.g.,
`./my_model_directory/preprocessor_config.json`.
@@ -150,12 +144,12 @@ class LayoutLMv2Processor:
**kwargs
) -> BatchEncoding:
"""
- This method first forwards the `images` argument to
- [`~LayoutLMv2FeatureExtractor.__call__`]. In case [`LayoutLMv2FeatureExtractor`] was
- initialized with `apply_ocr` set to `True`, it passes the obtained words and bounding boxes along with
- the additional arguments to [`~LayoutLMv2Tokenizer.__call__`] and returns the output, together
- with resized `images`. In case [`LayoutLMv2FeatureExtractor`] was initialized with `apply_ocr`
- set to `False`, it passes the words (`text`/``text_pair`) and `boxes` specified by the user along with the additional arguments to [`~LayoutLMv2Tokenizer.__call__`] and returns the output, together with resized `images``.
+ This method first forwards the `images` argument to [`~LayoutLMv2FeatureExtractor.__call__`]. In case
+ [`LayoutLMv2FeatureExtractor`] was initialized with `apply_ocr` set to `True`, it passes the obtained words and
+ bounding boxes along with the additional arguments to [`~LayoutLMv2Tokenizer.__call__`] and returns the output,
+ together with resized `images`. In case [`LayoutLMv2FeatureExtractor`] was initialized with `apply_ocr` set to
+ `False`, it passes the words (`text`/``text_pair`) and `boxes` specified by the user along with the additional
+ arguments to [`~LayoutLMv2Tokenizer.__call__`] and returns the output, together with resized `images``.
Please refer to the docstring of the above two methods for more information.
"""
diff --git a/src/transformers/models/layoutlmv2/tokenization_layoutlmv2.py b/src/transformers/models/layoutlmv2/tokenization_layoutlmv2.py
index 87057a325d..3f9816dd62 100644
--- a/src/transformers/models/layoutlmv2/tokenization_layoutlmv2.py
+++ b/src/transformers/models/layoutlmv2/tokenization_layoutlmv2.py
@@ -64,31 +64,31 @@ LAYOUTLMV2_ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING = r"""
padding (`bool`, `str` or [`~file_utils.PaddingStrategy`], *optional*, defaults to `False`):
Activates and controls padding. Accepts the following values:
- - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a
- single sequence if provided).
- - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the
- maximum acceptable input length for the model if that argument is not provided.
- - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of
- different lengths).
+ - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
+ sequence if provided).
+ - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
+ acceptable input length for the model if that argument is not provided.
+ - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different
+ lengths).
truncation (`bool`, `str` or [`~tokenization_utils_base.TruncationStrategy`], *optional*, defaults to `False`):
Activates and controls truncation. Accepts the following values:
- - `True` or `'longest_first'`: Truncate to a maximum length specified with the argument
- `max_length` or to the maximum acceptable input length for the model if that argument is not
- provided. This will truncate token by token, removing a token from the longest sequence in the pair
- if a pair of sequences (or a batch of pairs) is provided.
- - `'only_first'`: Truncate to a maximum length specified with the argument `max_length` or to
- the maximum acceptable input length for the model if that argument is not provided. This will only
+ - `True` or `'longest_first'`: Truncate to a maximum length specified with the argument `max_length` or
+ to the maximum acceptable input length for the model if that argument is not provided. This will
+ truncate token by token, removing a token from the longest sequence in the pair if a pair of
+ sequences (or a batch of pairs) is provided.
+ - `'only_first'`: Truncate to a maximum length specified with the argument `max_length` or to the
+ maximum acceptable input length for the model if that argument is not provided. This will only
truncate the first sequence of a pair if a pair of sequences (or a batch of pairs) is provided.
- - `'only_second'`: Truncate to a maximum length specified with the argument `max_length` or
- to the maximum acceptable input length for the model if that argument is not provided. This will only
+ - `'only_second'`: Truncate to a maximum length specified with the argument `max_length` or to the
+ maximum acceptable input length for the model if that argument is not provided. This will only
truncate the second sequence of a pair if a pair of sequences (or a batch of pairs) is provided.
- - `False` or `'do_not_truncate'` (default): No truncation (i.e., can output batch with
- sequence lengths greater than the model maximum admissible input size).
+ - `False` or `'do_not_truncate'` (default): No truncation (i.e., can output batch with sequence lengths
+ greater than the model maximum admissible input size).
max_length (`int`, *optional*):
Controls the maximum length to use by one of the truncation/padding parameters. If left unset or set to
- `None`, this will use the predefined model maximum length if a maximum length is required by one
- of the truncation/padding parameters. If the model has no specific maximum input length (like XLNet)
+ `None`, this will use the predefined model maximum length if a maximum length is required by one of the
+ truncation/padding parameters. If the model has no specific maximum input length (like XLNet)
truncation/padding to a maximum length will be deactivated.
stride (`int`, *optional*, defaults to 0):
If set to a number along with `max_length`, the overflowing tokens returned when
@@ -145,15 +145,15 @@ def subfinder(mylist, pattern):
class LayoutLMv2Tokenizer(PreTrainedTokenizer):
r"""
- Construct a LayoutLMv2 tokenizer. Based on WordPiece. [`LayoutLMv2Tokenizer`] can be used to
- turn words, word-level bounding boxes and optional word labels to token-level `input_ids`,
- `attention_mask`, `token_type_ids`, `bbox`, and optional `labels` (for token classification).
+ Construct a LayoutLMv2 tokenizer. Based on WordPiece. [`LayoutLMv2Tokenizer`] can be used to turn words, word-level
+ bounding boxes and optional word labels to token-level `input_ids`, `attention_mask`, `token_type_ids`, `bbox`, and
+ optional `labels` (for token classification).
- This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods.
- Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
+ this superclass for more information regarding those methods.
- [`LayoutLMv2Tokenizer`] runs end-to-end tokenization: punctuation splitting and wordpiece. It
- also turns the word-level bounding boxes into token-level bounding boxes.
+ [`LayoutLMv2Tokenizer`] runs end-to-end tokenization: punctuation splitting and wordpiece. It also turns the
+ word-level bounding boxes into token-level bounding boxes.
"""
@@ -335,8 +335,7 @@ class LayoutLMv2Tokenizer(PreTrainedTokenizer):
Optional second list of IDs for sequence pairs.
Returns:
- `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given
- sequence(s).
+ `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
"""
sep = [self.sep_token_id]
cls = [self.cls_token_id]
@@ -536,7 +535,6 @@ class LayoutLMv2Tokenizer(PreTrainedTokenizer):
verbose: bool = True,
**kwargs
) -> BatchEncoding:
- """ """
# Backward compatibility for 'truncation_strategy', 'pad_to_max_length'
padding_strategy, truncation_strategy, max_length, kwargs = self._get_padding_truncation_strategies(
@@ -720,9 +718,6 @@ class LayoutLMv2Tokenizer(PreTrainedTokenizer):
verbose: bool = True,
**kwargs
) -> List[int]:
- """
- ...
- """
encoded_inputs = self.encode_plus(
text=text,
text_pair=text_pair,
@@ -897,9 +892,9 @@ class LayoutLMv2Tokenizer(PreTrainedTokenizer):
*truncation_strategy = longest_first* or *True*, it is not possible to return overflowing tokens. Such a
combination of arguments will raise an error.
- Word-level `boxes` are turned into token-level `bbox`. If provided, word-level `word_labels` are
- turned into token-level `labels`. The word label is used for the first token of the word, while remaining
- tokens are labeled with -100, such that they will be ignored by the loss function.
+ Word-level `boxes` are turned into token-level `bbox`. If provided, word-level `word_labels` are turned into
+ token-level `labels`. The word label is used for the first token of the word, while remaining tokens are
+ labeled with -100, such that they will be ignored by the loss function.
Args:
text (`str`, `List[str]`, `List[List[str]]`):
@@ -1093,8 +1088,8 @@ class LayoutLMv2Tokenizer(PreTrainedTokenizer):
Args:
ids (`List[int]`):
- Tokenized input ids of the first sequence. Can be obtained from a string by chaining the `tokenize`
- and `convert_tokens_to_ids` methods.
+ Tokenized input ids of the first sequence. Can be obtained from a string by chaining the `tokenize` and
+ `convert_tokens_to_ids` methods.
token_boxes (`List[List[int]]`):
Bounding boxes of the first sequence.
pair_ids (`List[int]`, *optional*):
@@ -1109,26 +1104,26 @@ class LayoutLMv2Tokenizer(PreTrainedTokenizer):
truncation_strategy (`str` or [`~tokenization_utils_base.TruncationStrategy`], *optional*, defaults to `False`):
The strategy to follow for truncation. Can be:
- - `'longest_first'`: Truncate to a maximum length specified with the argument `max_length` or
- to the maximum acceptable input length for the model if that argument is not provided. This will
- truncate token by token, removing a token from the longest sequence in the pair if a pair of
- sequences (or a batch of pairs) is provided.
- - `'only_first'`: Truncate to a maximum length specified with the argument `max_length` or to
- the maximum acceptable input length for the model if that argument is not provided. This will only
+ - `'longest_first'`: Truncate to a maximum length specified with the argument `max_length` or to the
+ maximum acceptable input length for the model if that argument is not provided. This will truncate
+ token by token, removing a token from the longest sequence in the pair if a pair of sequences (or a
+ batch of pairs) is provided.
+ - `'only_first'`: Truncate to a maximum length specified with the argument `max_length` or to the
+ maximum acceptable input length for the model if that argument is not provided. This will only
truncate the first sequence of a pair if a pair of sequences (or a batch of pairs) is provided.
- - `'only_second'`: Truncate to a maximum length specified with the argument `max_length` or
- to the maximum acceptable input length for the model if that argument is not provided. This will only
+ - `'only_second'`: Truncate to a maximum length specified with the argument `max_length` or to the
+ maximum acceptable input length for the model if that argument is not provided. This will only
truncate the second sequence of a pair if a pair of sequences (or a batch of pairs) is provided.
- - `'do_not_truncate'` (default): No truncation (i.e., can output batch with sequence lengths
- greater than the model maximum admissible input size).
+ - `'do_not_truncate'` (default): No truncation (i.e., can output batch with sequence lengths greater
+ than the model maximum admissible input size).
stride (`int`, *optional*, defaults to 0):
If set to a positive number, the overflowing tokens returned will contain some tokens from the main
sequence returned. The value of this argument defines the number of additional tokens.
Returns:
- `Tuple[List[int], List[int], List[int]]`: The truncated `ids`, the truncated `pair_ids` and the
- list of overflowing tokens. Note: The *longest_first* strategy returns empty list of overflowing tokens if
- a pair of sequences (or a batch of pairs) is provided.
+ `Tuple[List[int], List[int], List[int]]`: The truncated `ids`, the truncated `pair_ids` and the list of
+ overflowing tokens. Note: The *longest_first* strategy returns empty list of overflowing tokens if a pair
+ of sequences (or a batch of pairs) is provided.
"""
if num_tokens_to_remove <= 0:
return ids, token_boxes, pair_ids, pair_token_boxes, labels, [], [], []
@@ -1214,7 +1209,8 @@ class LayoutLMv2Tokenizer(PreTrainedTokenizer):
Pad encoded inputs (on left/right and up to predefined length or max length in the batch)
Args:
- encoded_inputs: Dictionary of tokenized inputs (`List[int]`) or batch of tokenized inputs (`List[List[int]]`).
+ encoded_inputs:
+ Dictionary of tokenized inputs (`List[int]`) or batch of tokenized inputs (`List[List[int]]`).
max_length: maximum length of the returned list and optionally padding length (see below).
Will truncate by taking into account the special tokens.
padding_strategy: PaddingStrategy to use for padding.
@@ -1229,7 +1225,8 @@ class LayoutLMv2Tokenizer(PreTrainedTokenizer):
pad_to_multiple_of: (optional) Integer if set will pad the sequence to a multiple of the provided value.
This is especially useful to enable the use of Tensor Core on NVIDIA hardware with compute capability
>= 7.5 (Volta).
- return_attention_mask: (optional) Set to False to avoid returning attention mask (default: set to model specifics)
+ return_attention_mask:
+ (optional) Set to False to avoid returning attention mask (default: set to model specifics)
"""
# Load from model defaults
if return_attention_mask is None:
@@ -1299,7 +1296,8 @@ class BasicTokenizer(object):
tokenize_chinese_chars (`bool`, *optional*, defaults to `True`):
Whether or not to tokenize Chinese characters.
- This should likely be deactivated for Japanese (see this [issue](https://github.com/huggingface/transformers/issues/328)).
+ This should likely be deactivated for Japanese (see this
+ [issue](https://github.com/huggingface/transformers/issues/328)).
strip_accents: (`bool`, *optional*):
Whether or not to strip all accents. If this option is not specified, then it will be determined by the
value for `lowercase` (as in the original BERT).
diff --git a/src/transformers/models/layoutlmv2/tokenization_layoutlmv2_fast.py b/src/transformers/models/layoutlmv2/tokenization_layoutlmv2_fast.py
index cab5df57d8..0292f1a580 100644
--- a/src/transformers/models/layoutlmv2/tokenization_layoutlmv2_fast.py
+++ b/src/transformers/models/layoutlmv2/tokenization_layoutlmv2_fast.py
@@ -63,8 +63,8 @@ class LayoutLMv2TokenizerFast(PreTrainedTokenizerFast):
r"""
Construct a "fast" LayoutLMv2 tokenizer (backed by HuggingFace's *tokenizers* library). Based on WordPiece.
- This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main
- methods. Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should
+ refer to this superclass for more information regarding those methods.
Args:
vocab_file (`str`):
@@ -340,7 +340,6 @@ class LayoutLMv2TokenizerFast(PreTrainedTokenizerFast):
verbose: bool = True,
**kwargs
) -> BatchEncoding:
- """ """
# Backward compatibility for 'truncation_strategy', 'pad_to_max_length'
padding_strategy, truncation_strategy, max_length, kwargs = self._get_padding_truncation_strategies(
@@ -685,7 +684,8 @@ class LayoutLMv2TokenizerFast(PreTrainedTokenizerFast):
Pad encoded inputs (on left/right and up to predefined length or max length in the batch)
Args:
- encoded_inputs: Dictionary of tokenized inputs (`List[int]`) or batch of tokenized inputs (`List[List[int]]`).
+ encoded_inputs:
+ Dictionary of tokenized inputs (`List[int]`) or batch of tokenized inputs (`List[List[int]]`).
max_length: maximum length of the returned list and optionally padding length (see below).
Will truncate by taking into account the special tokens.
padding_strategy: PaddingStrategy to use for padding.
@@ -700,7 +700,8 @@ class LayoutLMv2TokenizerFast(PreTrainedTokenizerFast):
pad_to_multiple_of: (optional) Integer if set will pad the sequence to a multiple of the provided value.
This is especially useful to enable the use of Tensor Core on NVIDIA hardware with compute capability
>= 7.5 (Volta).
- return_attention_mask: (optional) Set to False to avoid returning attention mask (default: set to model specifics)
+ return_attention_mask:
+ (optional) Set to False to avoid returning attention mask (default: set to model specifics)
"""
# Load from model defaults
if return_attention_mask is None:
@@ -794,8 +795,7 @@ class LayoutLMv2TokenizerFast(PreTrainedTokenizerFast):
Optional second list of IDs for sequence pairs.
Returns:
- `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given
- sequence(s).
+ `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
"""
sep = [self.sep_token_id]
cls = [self.cls_token_id]
diff --git a/src/transformers/models/layoutxlm/processing_layoutxlm.py b/src/transformers/models/layoutxlm/processing_layoutxlm.py
index 59c7cb0e6c..4b0d27872d 100644
--- a/src/transformers/models/layoutxlm/processing_layoutxlm.py
+++ b/src/transformers/models/layoutxlm/processing_layoutxlm.py
@@ -32,20 +32,17 @@ class LayoutXLMProcessor:
[`LayoutXLMProcessor`] offers all the functionalities you need to prepare data for the model.
- It first uses [`LayoutLMv2FeatureExtractor`] to resize document images to a fixed size, and
- optionally applies OCR to get words and normalized bounding boxes. These are then provided to
- [`LayoutXLMTokenizer`] or [`LayoutXLMTokenizerFast`], which turns the words
- and bounding boxes into token-level `input_ids`, `attention_mask`, `token_type_ids`, `bbox`.
- Optionally, one can provide integer `word_labels`, which are turned into token-level `labels` for token
- classification tasks (such as FUNSD, CORD).
+ It first uses [`LayoutLMv2FeatureExtractor`] to resize document images to a fixed size, and optionally applies OCR
+ to get words and normalized bounding boxes. These are then provided to [`LayoutXLMTokenizer`] or
+ [`LayoutXLMTokenizerFast`], which turns the words and bounding boxes into token-level `input_ids`,
+ `attention_mask`, `token_type_ids`, `bbox`. Optionally, one can provide integer `word_labels`, which are turned
+ into token-level `labels` for token classification tasks (such as FUNSD, CORD).
Args:
feature_extractor (`LayoutLMv2FeatureExtractor`):
- An instance of [`LayoutLMv2FeatureExtractor`]. The feature extractor is a required
- input.
+ An instance of [`LayoutLMv2FeatureExtractor`]. The feature extractor is a required input.
tokenizer (`LayoutXLMTokenizer` or `LayoutXLMTokenizerFast`):
- An instance of [`LayoutXLMTokenizer`] or [`LayoutXLMTokenizerFast`].
- The tokenizer is a required input.
+ An instance of [`LayoutXLMTokenizer`] or [`LayoutXLMTokenizerFast`]. The tokenizer is a required input.
"""
def __init__(self, feature_extractor, tokenizer):
@@ -63,15 +60,14 @@ class LayoutXLMProcessor:
def save_pretrained(self, save_directory):
"""
- Save a LayoutXLM feature_extractor object and LayoutXLM tokenizer object to the directory `save_directory`,
- so that it can be re-loaded using the [`~LayoutXLMProcessor.from_pretrained`] class method.
+ Save a LayoutXLM feature_extractor object and LayoutXLM tokenizer object to the directory `save_directory`, so
+ that it can be re-loaded using the [`~LayoutXLMProcessor.from_pretrained`] class method.
- This class method is simply calling
- [`~feature_extraction_utils.FeatureExtractionMixin.save_pretrained`] and
- [`~tokenization_utils_base.PreTrainedTokenizer.save_pretrained`]. Please refer to the
- docstrings of the methods above for more information.
+ This class method is simply calling [`~feature_extraction_utils.FeatureExtractionMixin.save_pretrained`] and
+ [`~tokenization_utils_base.PreTrainedTokenizer.save_pretrained`]. Please refer to the docstrings of the methods
+ above for more information.
@@ -92,9 +88,9 @@ class LayoutXLMProcessor:
This class method is simply calling Layoutv2FeatureExtractor's
- [`~feature_extraction_utils.FeatureExtractionMixin.from_pretrained`] and
- LayoutXLMTokenizerFast's [`~tokenization_utils_base.PreTrainedTokenizer.from_pretrained`].
- Please refer to the docstrings of the methods above for more information.
+ [`~feature_extraction_utils.FeatureExtractionMixin.from_pretrained`] and LayoutXLMTokenizerFast's
+ [`~tokenization_utils_base.PreTrainedTokenizer.from_pretrained`]. Please refer to the docstrings of the methods
+ above for more information.
@@ -106,8 +102,7 @@ class LayoutXLMProcessor:
huggingface.co. Valid model ids can be located at the root-level, like `bert-base-uncased`, or
namespaced under a user or organization name, like `dbmdz/bert-base-german-cased`.
- a path to a *directory* containing a feature extractor file saved using the
- [`~SequenceFeatureExtractor.save_pretrained`] method, e.g.,
- `./my_model_directory/`.
+ [`~SequenceFeatureExtractor.save_pretrained`] method, e.g., `./my_model_directory/`.
- a path or url to a saved feature extractor JSON *file*, e.g.,
`./my_model_directory/preprocessor_config.json`.
@@ -150,12 +145,12 @@ class LayoutXLMProcessor:
**kwargs
) -> BatchEncoding:
"""
- This method first forwards the `images` argument to
- [`~LayoutLMv2FeatureExtractor.__call__`]. In case [`LayoutLMv2FeatureExtractor`] was
- initialized with `apply_ocr` set to `True`, it passes the obtained words and bounding boxes along with
- the additional arguments to [`~LayoutXLMTokenizer.__call__`] and returns the output, together
- with resized `images`. In case [`LayoutLMv2FeatureExtractor`] was initialized with `apply_ocr`
- set to `False`, it passes the words (`text`/``text_pair`) and `boxes` specified by the user along with the additional arguments to [`~LayoutXLMTokenizer.__call__`] and returns the output, together with resized `images``.
+ This method first forwards the `images` argument to [`~LayoutLMv2FeatureExtractor.__call__`]. In case
+ [`LayoutLMv2FeatureExtractor`] was initialized with `apply_ocr` set to `True`, it passes the obtained words and
+ bounding boxes along with the additional arguments to [`~LayoutXLMTokenizer.__call__`] and returns the output,
+ together with resized `images`. In case [`LayoutLMv2FeatureExtractor`] was initialized with `apply_ocr` set to
+ `False`, it passes the words (`text`/``text_pair`) and `boxes` specified by the user along with the additional
+ arguments to [`~LayoutXLMTokenizer.__call__`] and returns the output, together with resized `images``.
Please refer to the docstring of the above two methods for more information.
"""
diff --git a/src/transformers/models/layoutxlm/tokenization_layoutxlm.py b/src/transformers/models/layoutxlm/tokenization_layoutxlm.py
index 75cc9a591c..6928454f64 100644
--- a/src/transformers/models/layoutxlm/tokenization_layoutxlm.py
+++ b/src/transformers/models/layoutxlm/tokenization_layoutxlm.py
@@ -50,8 +50,8 @@ class LayoutXLMTokenizer(PreTrainedTokenizer):
Adapted from [`RobertaTokenizer`] and [`XLNetTokenizer`]. Based on
[SentencePiece](https://github.com/google/sentencepiece).
- This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods.
- Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
+ this superclass for more information regarding those methods.
Args:
vocab_file (`str`):
@@ -71,8 +71,8 @@ class LayoutXLMTokenizer(PreTrainedTokenizer):
- When building a sequence using special tokens, this is not the token that is used for the end of
- sequence. The token used is the `sep_token`.
+ When building a sequence using special tokens, this is not the token that is used for the end of sequence.
+ The token used is the `sep_token`.
@@ -105,7 +105,9 @@ class LayoutXLMTokenizer(PreTrainedTokenizer):
additional_special_tokens (`List[str]`, *optional*, defaults to `["NOTUSED", "NOTUSED"]`):
Additional special tokens used by the tokenizer.
sp_model_kwargs (`dict`, *optional*):
- Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things, to set:
+ Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for
+ SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things,
+ to set:
- `enable_sampling`: Enable subword regularization.
- `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout.
@@ -689,9 +691,9 @@ class LayoutXLMTokenizer(PreTrainedTokenizer):
truncates sequences if overflowing while taking into account the special tokens and manages a moving window
(with user defined stride) for overflowing tokens.
- Word-level `boxes` are turned into token-level `bbox`. If provided, word-level `word_labels` are
- turned into token-level `labels`. The word label is used for the first token of the word, while remaining
- tokens are labeled with -100, such that they will be ignored by the loss function.
+ Word-level `boxes` are turned into token-level `bbox`. If provided, word-level `word_labels` are turned into
+ token-level `labels`. The word label is used for the first token of the word, while remaining tokens are
+ labeled with -100, such that they will be ignored by the loss function.
Args:
text (`str`, `List[str]`, `List[List[str]]`):
@@ -874,8 +876,8 @@ class LayoutXLMTokenizer(PreTrainedTokenizer):
Args:
ids (`List[int]`):
- Tokenized input ids of the first sequence. Can be obtained from a string by chaining the `tokenize`
- and `convert_tokens_to_ids` methods.
+ Tokenized input ids of the first sequence. Can be obtained from a string by chaining the `tokenize` and
+ `convert_tokens_to_ids` methods.
token_boxes (`List[List[int]]`):
Bounding boxes of the first sequence.
pair_ids (`List[int]`, *optional*):
@@ -890,25 +892,25 @@ class LayoutXLMTokenizer(PreTrainedTokenizer):
truncation_strategy (`str` or [`~tokenization_utils_base.TruncationStrategy`], *optional*, defaults to `False`):
The strategy to follow for truncation. Can be:
- - `'longest_first'`: Truncate to a maximum length specified with the argument `max_length` or
- to the maximum acceptable input length for the model if that argument is not provided. This will
- truncate token by token, removing a token from the longest sequence in the pair if a pair of
- sequences (or a batch of pairs) is provided.
- - `'only_first'`: Truncate to a maximum length specified with the argument `max_length` or to
- the maximum acceptable input length for the model if that argument is not provided. This will only
+ - `'longest_first'`: Truncate to a maximum length specified with the argument `max_length` or to the
+ maximum acceptable input length for the model if that argument is not provided. This will truncate
+ token by token, removing a token from the longest sequence in the pair if a pair of sequences (or a
+ batch of pairs) is provided.
+ - `'only_first'`: Truncate to a maximum length specified with the argument `max_length` or to the
+ maximum acceptable input length for the model if that argument is not provided. This will only
truncate the first sequence of a pair if a pair of sequences (or a batch of pairs) is provided.
- - `'only_second'`: Truncate to a maximum length specified with the argument `max_length` or
- to the maximum acceptable input length for the model if that argument is not provided. This will only
+ - `'only_second'`: Truncate to a maximum length specified with the argument `max_length` or to the
+ maximum acceptable input length for the model if that argument is not provided. This will only
truncate the second sequence of a pair if a pair of sequences (or a batch of pairs) is provided.
- - `'do_not_truncate'` (default): No truncation (i.e., can output batch with sequence lengths
- greater than the model maximum admissible input size).
+ - `'do_not_truncate'` (default): No truncation (i.e., can output batch with sequence lengths greater
+ than the model maximum admissible input size).
stride (`int`, *optional*, defaults to 0):
If set to a positive number, the overflowing tokens returned will contain some tokens from the main
sequence returned. The value of this argument defines the number of additional tokens.
Returns:
- `Tuple[List[int], List[int], List[int]]`: The truncated `ids`, the truncated `pair_ids` and the
- list of overflowing tokens.
+ `Tuple[List[int], List[int], List[int]]`: The truncated `ids`, the truncated `pair_ids` and the list of
+ overflowing tokens.
"""
if num_tokens_to_remove <= 0:
return ids, token_boxes, pair_ids, pair_token_boxes, labels, [], [], []
@@ -995,7 +997,8 @@ class LayoutXLMTokenizer(PreTrainedTokenizer):
Pad encoded inputs (on left/right and up to predefined length or max length in the batch)
Args:
- encoded_inputs: Dictionary of tokenized inputs (`List[int]`) or batch of tokenized inputs (`List[List[int]]`).
+ encoded_inputs:
+ Dictionary of tokenized inputs (`List[int]`) or batch of tokenized inputs (`List[List[int]]`).
max_length: maximum length of the returned list and optionally padding length (see below).
Will truncate by taking into account the special tokens.
padding_strategy: PaddingStrategy to use for padding.
@@ -1010,7 +1013,8 @@ class LayoutXLMTokenizer(PreTrainedTokenizer):
pad_to_multiple_of: (optional) Integer if set will pad the sequence to a multiple of the provided value.
This is especially useful to enable the use of Tensor Core on NVIDIA hardware with compute capability
>= 7.5 (Volta).
- return_attention_mask: (optional) Set to False to avoid returning attention mask (default: set to model specifics)
+ return_attention_mask:
+ (optional) Set to False to avoid returning attention mask (default: set to model specifics)
"""
# Load from model defaults
if return_attention_mask is None:
diff --git a/src/transformers/models/layoutxlm/tokenization_layoutxlm_fast.py b/src/transformers/models/layoutxlm/tokenization_layoutxlm_fast.py
index 8c17828de9..47dd362505 100644
--- a/src/transformers/models/layoutxlm/tokenization_layoutxlm_fast.py
+++ b/src/transformers/models/layoutxlm/tokenization_layoutxlm_fast.py
@@ -53,10 +53,11 @@ logger = logging.get_logger(__name__)
class LayoutXLMTokenizerFast(PreTrainedTokenizerFast):
"""
Construct a "fast" LayoutXLM tokenizer (backed by HuggingFace's *tokenizers* library). Adapted from
- [`RobertaTokenizer`] and [`XLNetTokenizer`]. Based on [BPE](https://huggingface.co/docs/tokenizers/python/latest/components.html?highlight=BPE#models).
+ [`RobertaTokenizer`] and [`XLNetTokenizer`]. Based on
+ [BPE](https://huggingface.co/docs/tokenizers/python/latest/components.html?highlight=BPE#models).
- This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main
- methods. Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should
+ refer to this superclass for more information regarding those methods.
Args:
vocab_file (`str`):
@@ -76,8 +77,8 @@ class LayoutXLMTokenizerFast(PreTrainedTokenizerFast):
- When building a sequence using special tokens, this is not the token that is used for the end of
- sequence. The token used is the `sep_token`.
+ When building a sequence using special tokens, this is not the token that is used for the end of sequence.
+ The token used is the `sep_token`.
@@ -558,7 +559,8 @@ class LayoutXLMTokenizerFast(PreTrainedTokenizerFast):
Pad encoded inputs (on left/right and up to predefined length or max length in the batch)
Args:
- encoded_inputs: Dictionary of tokenized inputs (`List[int]`) or batch of tokenized inputs (`List[List[int]]`).
+ encoded_inputs:
+ Dictionary of tokenized inputs (`List[int]`) or batch of tokenized inputs (`List[List[int]]`).
max_length: maximum length of the returned list and optionally padding length (see below).
Will truncate by taking into account the special tokens.
padding_strategy: PaddingStrategy to use for padding.
@@ -573,7 +575,8 @@ class LayoutXLMTokenizerFast(PreTrainedTokenizerFast):
pad_to_multiple_of: (optional) Integer if set will pad the sequence to a multiple of the provided value.
This is especially useful to enable the use of Tensor Core on NVIDIA hardware with compute capability
>= 7.5 (Volta).
- return_attention_mask: (optional) Set to False to avoid returning attention mask (default: set to model specifics)
+ return_attention_mask:
+ (optional) Set to False to avoid returning attention mask (default: set to model specifics)
"""
# Load from model defaults
if return_attention_mask is None:
diff --git a/src/transformers/models/led/configuration_led.py b/src/transformers/models/led/configuration_led.py
index e6b617cc24..5f534ab287 100644
--- a/src/transformers/models/led/configuration_led.py
+++ b/src/transformers/models/led/configuration_led.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" LED model configuration """
+""" LED model configuration"""
from typing import List, Union
@@ -30,12 +30,13 @@ LED_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class LEDConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`LEDModel`]. It is used to
- instantiate an LED model according to the specified arguments, defining the model architecture. Instantiating a
- configuration with the defaults will yield a similar configuration to that of the LED [allenai/led-base-16384](https://huggingface.co/allenai/led-base-16384) architecture.
+ This is the configuration class to store the configuration of a [`LEDModel`]. It is used to instantiate an LED
+ model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
+ defaults will yield a similar configuration to that of the LED
+ [allenai/led-base-16384](https://huggingface.co/allenai/led-base-16384) architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
@@ -57,8 +58,8 @@ class LEDConfig(PretrainedConfig):
encoder_ffn_dim (`int`, *optional*, defaults to 4096):
Dimensionality of the "intermediate" (often named feed-forward) layer in decoder.
activation_function (`str` or `function`, *optional*, defaults to `"gelu"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"silu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"silu"` and `"gelu_new"` are supported.
dropout (`float`, *optional*, defaults to 0.1):
The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
attention_dropout (`float`, *optional*, defaults to 0.0):
@@ -74,11 +75,11 @@ class LEDConfig(PretrainedConfig):
init_std (`float`, *optional*, defaults to 0.02):
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
encoder_layerdrop: (`float`, *optional*, defaults to 0.0):
- The LayerDrop probability for the encoder. See the [LayerDrop paper](see
- https://arxiv.org/abs/1909.11556) for more details.
+ The LayerDrop probability for the encoder. See the [LayerDrop paper](see https://arxiv.org/abs/1909.11556)
+ for more details.
decoder_layerdrop: (`float`, *optional*, defaults to 0.0):
- The LayerDrop probability for the decoder. See the [LayerDrop paper](see
- https://arxiv.org/abs/1909.11556) for more details.
+ The LayerDrop probability for the decoder. See the [LayerDrop paper](see https://arxiv.org/abs/1909.11556)
+ for more details.
use_cache (`bool`, *optional*, defaults to `True`):
Whether or not the model should return the last key/values attentions (not used by all models)
@@ -90,14 +91,12 @@ class LEDConfig(PretrainedConfig):
>>> from transformers import LEDModel, LEDConfig
- >>> # Initializing a LED allenai/led-base-16384 style configuration
- >>> configuration = LEDConfig()
+ >>> # Initializing a LED allenai/led-base-16384 style configuration >>> configuration = LEDConfig()
- >>> # Initializing a model from the allenai/led-base-16384 style configuration
- >>> model = LEDModel(configuration)
+ >>> # Initializing a model from the allenai/led-base-16384 style configuration >>> model =
+ LEDModel(configuration)
- >>> # Accessing the model configuration
- >>> configuration = model.config
+ >>> # Accessing the model configuration >>> configuration = model.config
"""
model_type = "led"
attribute_map = {
diff --git a/src/transformers/models/led/modeling_led.py b/src/transformers/models/led/modeling_led.py
index 1610b67ada..3f6fb9d00a 100755
--- a/src/transformers/models/led/modeling_led.py
+++ b/src/transformers/models/led/modeling_led.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" PyTorch LED model. """
+""" PyTorch LED model."""
import math
@@ -350,11 +350,9 @@ class LEDEncoderSelfAttention(nn.Module):
window_overlap = num_rows = 4
```
- (pad & diagonalize) =>
- [ 0.4983, 2.6918, -0.0071, 1.0492, 0.0000, 0.0000, 0.0000
- 0.0000, -1.8348, 0.7672, 0.2986, 0.0285, 0.0000, 0.0000
- 0.0000, 0.0000, -0.7584, 0.4206, -0.0405, 0.1599, 0.0000
- 0.0000, 0.0000, 0.0000, 2.0514, -1.1600, 0.5372, 0.2629 ]
+ (pad & diagonalize) => [ 0.4983, 2.6918, -0.0071, 1.0492, 0.0000, 0.0000, 0.0000
+ 0.0000, -1.8348, 0.7672, 0.2986, 0.0285, 0.0000, 0.0000 0.0000, 0.0000, -0.7584, 0.4206,
+ -0.0405, 0.1599, 0.0000 0.0000, 0.0000, 0.0000, 2.0514, -1.1600, 0.5372, 0.2629 ]
"""
total_num_heads, num_chunks, window_overlap, hidden_dim = chunked_hidden_states.size()
chunked_hidden_states = nn.functional.pad(
@@ -982,7 +980,8 @@ class LEDDecoderLayer(nn.Module):
hidden_states (`torch.FloatTensor`): input to the layer of shape *(seq_len, batch, embed_dim)*
attention_mask (`torch.FloatTensor`): attention mask of size
*(batch, 1, tgt_len, src_len)* where padding elements are indicated by very large negative values.
- encoder_hidden_states (`torch.FloatTensor`): cross attention input to the layer of shape *(seq_len, batch, embed_dim)*
+ encoder_hidden_states (`torch.FloatTensor`):
+ cross attention input to the layer of shape *(seq_len, batch, embed_dim)*
encoder_attention_mask (`torch.FloatTensor`): encoder attention mask of size
*(batch, 1, tgt_len, src_len)* where padding elements are indicated by very large negative values.
layer_head_mask (`torch.FloatTensor`): mask for attention heads in a given layer of size
@@ -1118,26 +1117,28 @@ class LEDEncoderBaseModelOutput(ModelOutput):
last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
Sequence of hidden-states at the output of the last layer of the model.
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x + attention_window + 1)`, where `x` is the number of tokens with global attention
- mask.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x +
+ attention_window + 1)`, where `x` is the number of tokens with global attention mask.
Local attentions weights after the attention softmax, used to compute the weighted average in the
self-attention heads. Those are the attention weights from every token in the sequence to every token with
- global attention (first `x` values) and to every token in the attention window (remaining
- `attention_window + 1` values). Note that the first `x` values refer to tokens with fixed positions in
- the text, but the remaining `attention_window + 1` values refer to tokens with relative positions: the
- attention weight of a token to itself is located at index `x + attention_window / 2` and the
- `attention_window / 2` preceding (succeeding) values are the attention weights to the `attention_window / 2` preceding (succeeding) tokens. If the attention window contains a token with global attention, the
- attention weight at the corresponding index is set to 0; the value should be accessed from the first `x`
- attention weights. If a token has global attention, the attention weights to all other tokens in
- `attentions` is set to 0, the values should be accessed from `global_attentions`.
+ global attention (first `x` values) and to every token in the attention window (remaining `attention_window
+ + 1` values). Note that the first `x` values refer to tokens with fixed positions in the text, but the
+ remaining `attention_window + 1` values refer to tokens with relative positions: the attention weight of a
+ token to itself is located at index `x + attention_window / 2` and the `attention_window / 2` preceding
+ (succeeding) values are the attention weights to the `attention_window / 2` preceding (succeeding) tokens.
+ If the attention window contains a token with global attention, the attention weight at the corresponding
+ index is set to 0; the value should be accessed from the first `x` attention weights. If a token has global
+ attention, the attention weights to all other tokens in `attentions` is set to 0, the values should be
+ accessed from `global_attentions`.
global_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x)`, where `x` is the number of tokens with global attention mask.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x)`,
+ where `x` is the number of tokens with global attention mask.
Global attentions weights after the attention softmax, used to compute the weighted average in the
self-attention heads. Those are the attention weights from every token with global attention to every token
@@ -1160,41 +1161,47 @@ class LEDSeq2SeqModelOutput(ModelOutput):
last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
Sequence of hidden-states at the output of the last layer of the decoder of the model.
- If `past_key_values` is used only the last hidden-state of the sequences of shape `(batch_size, 1, hidden_size)` is output.
+ If `past_key_values` is used only the last hidden-state of the sequences of shape `(batch_size, 1,
+ hidden_size)` is output.
past_key_values (`List[torch.FloatTensor]`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- List of `torch.FloatTensor` of length `config.n_layers`, with each tensor of shape `(2, batch_size, num_heads, sequence_length, embed_size_per_head)`).
+ List of `torch.FloatTensor` of length `config.n_layers`, with each tensor of shape `(2, batch_size,
+ num_heads, sequence_length, embed_size_per_head)`).
Contains pre-computed hidden-states (key and values in the attention blocks) of the decoder that can be
used (see `past_key_values` input) to speed up sequential decoding.
decoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the decoder at the output of each layer plus the initial embedding outputs.
decoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the decoder, after the attention softmax, used to compute the weighted average in the
self-attention heads.
cross_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the decoder's cross-attention layer, after the attention softmax, used to compute the
weighted average in the cross-attention heads.
encoder_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
Sequence of hidden-states at the output of the last layer of the encoder of the model.
encoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the encoder at the output of each layer plus the initial embedding outputs.
encoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the encoder, after the attention softmax, used to compute the weighted average in the
self-attention heads.
encoder_global_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x)`, where `x` is the number of tokens with global attention mask.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x)`,
+ where `x` is the number of tokens with global attention mask.
Global attentions weights after the attention softmax, used to compute the weighted average in the
self-attention heads. Those are the attention weights from every token with global attention to every token
@@ -1223,39 +1230,44 @@ class LEDSeq2SeqLMOutput(ModelOutput):
logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
past_key_values (`List[torch.FloatTensor]`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- List of `torch.FloatTensor` of length `config.n_layers`, with each tensor of shape `(2, batch_size, num_heads, sequence_length, embed_size_per_head)`).
+ List of `torch.FloatTensor` of length `config.n_layers`, with each tensor of shape `(2, batch_size,
+ num_heads, sequence_length, embed_size_per_head)`).
Contains pre-computed hidden-states (key and values in the attention blocks) of the decoder that can be
used (see `past_key_values` input) to speed up sequential decoding.
decoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the decoder at the output of each layer plus the initial embedding outputs.
decoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the decoder, after the attention softmax, used to compute the weighted average in the
self-attention heads.
cross_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the decoder's cross-attention layer, after the attention softmax, used to compute the
weighted average in the cross-attention heads.
encoder_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
Sequence of hidden-states at the output of the last layer of the encoder of the model.
encoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the encoder at the output of each layer plus the initial embedding outputs.
encoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the encoder, after the attention softmax, used to compute the weighted average in the
self-attention heads.
encoder_global_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x)`, where `x` is the number of tokens with global attention mask.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x)`,
+ where `x` is the number of tokens with global attention mask.
Global attentions weights after the attention softmax, used to compute the weighted average in the
self-attention heads. Those are the attention weights from every token with global attention to every token
@@ -1285,39 +1297,44 @@ class LEDSeq2SeqSequenceClassifierOutput(ModelOutput):
logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
Classification (or regression if config.num_labels==1) scores (before SoftMax).
past_key_values (`List[torch.FloatTensor]`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- List of `torch.FloatTensor` of length `config.n_layers`, with each tensor of shape `(2, batch_size, num_heads, sequence_length, embed_size_per_head)`).
+ List of `torch.FloatTensor` of length `config.n_layers`, with each tensor of shape `(2, batch_size,
+ num_heads, sequence_length, embed_size_per_head)`).
Contains pre-computed hidden-states (key and values in the attention blocks) of the decoder that can be
used (see `past_key_values` input) to speed up sequential decoding.
decoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the decoder at the output of each layer plus the initial embedding outputs.
decoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the decoder, after the attention softmax, used to compute the weighted average in the
self-attention heads.
cross_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the decoder's cross-attention layer, after the attention softmax, used to compute the
weighted average in the cross-attention heads.
encoder_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
Sequence of hidden-states at the output of the last layer of the encoder of the model.
encoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the encoder at the output of each layer plus the initial embedding outputs.
encoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the encoder, after the attention softmax, used to compute the weighted average in the
self-attention heads.
encoder_global_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x)`, where `x` is the number of tokens with global attention mask.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x)`,
+ where `x` is the number of tokens with global attention mask.
Global attentions weights after the attention softmax, used to compute the weighted average in the
self-attention heads. Those are the attention weights from every token with global attention to every token
@@ -1349,39 +1366,44 @@ class LEDSeq2SeqQuestionAnsweringModelOutput(ModelOutput):
end_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length)`):
Span-end scores (before SoftMax).
past_key_values (`List[torch.FloatTensor]`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- List of `torch.FloatTensor` of length `config.n_layers`, with each tensor of shape `(2, batch_size, num_heads, sequence_length, embed_size_per_head)`).
+ List of `torch.FloatTensor` of length `config.n_layers`, with each tensor of shape `(2, batch_size,
+ num_heads, sequence_length, embed_size_per_head)`).
Contains pre-computed hidden-states (key and values in the attention blocks) of the decoder that can be
used (see `past_key_values` input) to speed up sequential decoding.
decoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the decoder at the output of each layer plus the initial embedding outputs.
decoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the decoder, after the attention softmax, used to compute the weighted average in the
self-attention heads.
cross_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the decoder's cross-attention layer, after the attention softmax, used to compute the
weighted average in the cross-attention heads.
encoder_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
Sequence of hidden-states at the output of the last layer of the encoder of the model.
encoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the encoder at the output of each layer plus the initial embedding outputs.
encoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the encoder, after the attention softmax, used to compute the weighted average in the
self-attention heads.
encoder_global_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x)`, where `x` is the number of tokens with global attention mask.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x)`,
+ where `x` is the number of tokens with global attention mask.
Global attentions weights after the attention softmax, used to compute the weighted average in the
self-attention heads. Those are the attention weights from every token with global attention to every token
@@ -1402,13 +1424,13 @@ class LEDSeq2SeqQuestionAnsweringModelOutput(ModelOutput):
LED_START_DOCSTRING = r"""
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
Parameters:
config ([`LEDConfig`]):
@@ -1420,43 +1442,34 @@ LED_START_DOCSTRING = r"""
LED_GENERATION_EXAMPLE = r"""
Summarization example::
- >>> import torch
- >>> from transformers import LEDTokenizer, LEDForConditionalGeneration
+ >>> import torch >>> from transformers import LEDTokenizer, LEDForConditionalGeneration
- >>> model = LEDForConditionalGeneration.from_pretrained('allenai/led-large-16384-arxiv')
- >>> tokenizer = LEDTokenizer.from_pretrained('allenai/led-large-16384-arxiv')
+ >>> model = LEDForConditionalGeneration.from_pretrained('allenai/led-large-16384-arxiv') >>> tokenizer =
+ LEDTokenizer.from_pretrained('allenai/led-large-16384-arxiv')
- >>> ARTICLE_TO_SUMMARIZE = '''Transformers (Vaswani et al., 2017) have achieved state-of-the-art
- ... results in a wide range of natural language tasks including generative
- ... language modeling (Dai et al., 2019; Radford et al., 2019) and discriminative
- ... language understanding (Devlin et al., 2019). This success is partly due to
- ... the self-attention component which enables the network to capture contextual
- ... information from the entire sequence. While powerful, the memory and computational
- ... requirements of self-attention grow quadratically with sequence length, making
- ... it infeasible (or very expensive) to process long sequences.
- ...
- ... To address this limitation, we present Longformer, a modified Transformer
- ... architecture with a self-attention operation that scales linearly with the
- ... sequence length, making it versatile for processing long documents (Fig 1). This
- ... is an advantage for natural language tasks such as long document classification,
- ... question answering (QA), and coreference resolution, where existing approaches
- ... partition or shorten the long context into smaller sequences that fall within the
- ... typical 512 token limit of BERT-style pretrained models. Such partitioning could
- ... potentially result in loss of important cross-partition information, and to
- ... mitigate this problem, existing methods often rely on complex architectures to
- ... address such interactions. On the other hand, our proposed Longformer is able to
- ... build contextual representations of the entire context using multiple layers of
- ... attention, reducing the need for task-specific architectures.'''
- >>> inputs = tokenizer.encode(ARTICLE_TO_SUMMARIZE, return_tensors='pt')
+ >>> ARTICLE_TO_SUMMARIZE = '''Transformers (Vaswani et al., 2017) have achieved state-of-the-art ... results in
+ a wide range of natural language tasks including generative ... language modeling (Dai et al., 2019; Radford et
+ al., 2019) and discriminative ... language understanding (Devlin et al., 2019). This success is partly due to
+ ... the self-attention component which enables the network to capture contextual ... information from the
+ entire sequence. While powerful, the memory and computational ... requirements of self-attention grow
+ quadratically with sequence length, making ... it infeasible (or very expensive) to process long sequences. ...
+ ... To address this limitation, we present Longformer, a modified Transformer ... architecture with a
+ self-attention operation that scales linearly with the ... sequence length, making it versatile for processing
+ long documents (Fig 1). This ... is an advantage for natural language tasks such as long document
+ classification, ... question answering (QA), and coreference resolution, where existing approaches ...
+ partition or shorten the long context into smaller sequences that fall within the ... typical 512 token limit
+ of BERT-style pretrained models. Such partitioning could ... potentially result in loss of important
+ cross-partition information, and to ... mitigate this problem, existing methods often rely on complex
+ architectures to ... address such interactions. On the other hand, our proposed Longformer is able to ... build
+ contextual representations of the entire context using multiple layers of ... attention, reducing the need for
+ task-specific architectures.''' >>> inputs = tokenizer.encode(ARTICLE_TO_SUMMARIZE, return_tensors='pt')
- >>> # Global attention on the first token (cf. Beltagy et al. 2020)
- >>> global_attention_mask = torch.zeros_like(inputs)
- >>> global_attention_mask[:, 0] = 1
+ >>> # Global attention on the first token (cf. Beltagy et al. 2020) >>> global_attention_mask =
+ torch.zeros_like(inputs) >>> global_attention_mask[:, 0] = 1
- >>> # Generate Summary
- >>> summary_ids = model.generate(inputs, global_attention_mask=global_attention_mask,
- ... num_beams=3, max_length=32, early_stopping=True)
- >>> print(tokenizer.decode(summary_ids[0], skip_special_tokens=True, clean_up_tokenization_spaces=True))
+ >>> # Generate Summary >>> summary_ids = model.generate(inputs, global_attention_mask=global_attention_mask,
+ ... num_beams=3, max_length=32, early_stopping=True) >>> print(tokenizer.decode(summary_ids[0],
+ skip_special_tokens=True, clean_up_tokenization_spaces=True))
"""
LED_INPUTS_DOCSTRING = r"""
@@ -1465,9 +1478,8 @@ LED_INPUTS_DOCSTRING = r"""
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
it.
- Indices can be obtained using [`LEDTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`LEDTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -1480,28 +1492,27 @@ LED_INPUTS_DOCSTRING = r"""
decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
Indices of decoder input sequence tokens in the vocabulary.
- Indices can be obtained using [`LedTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`LedTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
- LED uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If
- `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
- `past_key_values`).
+ LED uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values`
+ is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`).
decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
- Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will
- also be used by default.
+ Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
+ be used by default.
- If you want to change padding behavior, you should read [`modeling_led._prepare_decoder_inputs`] and
- modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more
- information on the default strategy.
+ If you want to change padding behavior, you should read [`modeling_led._prepare_decoder_inputs`] and modify
+ to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more information on the
+ default strategy.
global_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
Mask to decide the attention given on each token, local attention or global attention for the encoder.
Tokens with global attention attends to all other tokens, and all other tokens attend to them. This is
important for task-specific finetuning because it makes the model more flexible at representing the task.
For example, for classification, the token should be given global attention. For QA, all question
- tokens should also have global attention. Please refer to the [Longformer paper](https://arxiv.org/abs/2004.05150) for more details. Mask values selected in `[0, 1]`:
+ tokens should also have global attention. Please refer to the [Longformer
+ paper](https://arxiv.org/abs/2004.05150) for more details. Mask values selected in `[0, 1]`:
- 0 for local attention (a sliding window attention),
- 1 for global attention (tokens that attend to all other tokens, and all other tokens attend to them).
@@ -1517,39 +1528,42 @@ LED_INPUTS_DOCSTRING = r"""
- 1 indicates the head is **not masked**,
- 0 indicates the head is **masked**.
cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*):
- Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, 1]`:
+ Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0,
+ 1]`:
- 1 indicates the head is **not masked**,
- 0 indicates the head is **masked**.
encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*):
- Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*:
- `attentions`) `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`,
- *optional*) is a sequence of hidden-states at the output of the last layer of the encoder. Used in the
- cross-attention of the decoder.
+ Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`)
+ `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of
+ hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder.
past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors
- of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
- shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
+ Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
+ `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape
+ `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all ``decoder_input_ids``` of shape `(batch_size, sequence_length)`. inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ ``decoder_input_ids``` of shape `(batch_size, sequence_length)`. inputs_embeds (`torch.FloatTensor` of
+ shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids`
+ you can choose to directly pass an embedded representation. This is useful if you want more control over
+ how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup
+ matrix.
decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*):
Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded
- representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds`
- have to be input (see `past_key_values`). This is useful if you want more control over how to convert
+ representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be
+ input (see `past_key_values`). This is useful if you want more control over how to convert
`decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix.
- If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds`
- takes the value of `inputs_embeds`.
+ If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds` takes the value
+ of `inputs_embeds`.
use_cache (`bool`, *optional*):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`).
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`).
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -1678,9 +1692,8 @@ class LEDEncoder(LEDPreTrainedModel):
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
provide it.
- Indices can be obtained using [`LEDTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`]
- for details.
+ Indices can be obtained using [`LEDTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -1695,7 +1708,8 @@ class LEDEncoder(LEDPreTrainedModel):
Tokens with global attention attends to all other tokens, and all other tokens attend to them. This is
important for task-specific finetuning because it makes the model more flexible at representing the
task. For example, for classification, the token should be given global attention. For QA, all
- question tokens should also have global attention. Please refer to the [Longformer paper](https://arxiv.org/abs/2004.05150) for more details. Mask values selected in `[0, 1]`:
+ question tokens should also have global attention. Please refer to the [Longformer
+ paper](https://arxiv.org/abs/2004.05150) for more details. Mask values selected in `[0, 1]`:
- 0 for local attention (a sliding window attention),
- 1 for global attention (tokens that attend to all other tokens, and all other tokens attend to them).
@@ -1705,9 +1719,9 @@ class LEDEncoder(LEDPreTrainedModel):
- 1 indicates the head is **not masked**,
- 0 indicates the head is **masked**.
inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded
- representation. This is useful if you want more control over how to convert `input_ids` indices
- into associated vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
+ This is useful if you want more control over how to convert `input_ids` indices into associated vectors
+ than the model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under
returned tensors for more detail.
@@ -1904,9 +1918,8 @@ class LEDDecoder(LEDPreTrainedModel):
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
provide it.
- Indices can be obtained using [`LEDTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`]
- for details.
+ Indices can be obtained using [`LEDTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -1921,7 +1934,8 @@ class LEDDecoder(LEDPreTrainedModel):
global attention attends to all other tokens, and all other tokens attend to them. This is important
for task-specific finetuning because it makes the model more flexible at representing the task. For
example, for classification, the token should be given global attention. For QA, all question
- tokens should also have global attention. Please refer to the [Longformer paper](https://arxiv.org/abs/2004.05150) for more details. Mask values selected in `[0, 1]`:
+ tokens should also have global attention. Please refer to the [Longformer
+ paper](https://arxiv.org/abs/2004.05150) for more details. Mask values selected in `[0, 1]`:
- 0 for local attention (a sliding window attention),
- 1 for global attention (tokens that attend to all other tokens, and all other tokens attend to them).
@@ -1949,19 +1963,20 @@ class LEDDecoder(LEDPreTrainedModel):
- 0 indicates the head is **masked**.
past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2
- tensors of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional
- tensors of shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
+ Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of
+ shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
+ shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
Contains pre-computed hidden-states (key and values in the self-attention blocks and in the
- cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential
- decoding.
+ cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
- If `past_key_values` are used, the user can optionally input only the last
- `decoder_input_ids` (those that don't have their past key value states given to this model) of
- shape `(batch_size, 1)` instead of all ``decoder_input_ids``` of shape `(batch_size,
- sequence_length)`. inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices
- into associated vectors than the model's internal embedding lookup matrix.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those
+ that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of
+ all ``decoder_input_ids``` of shape `(batch_size, sequence_length)`. inputs_embeds (`torch.FloatTensor`
+ of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing
+ `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more
+ control over how to convert `input_ids` indices into associated vectors than the model's internal
+ embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under
returned tensors for more detail.
@@ -2309,7 +2324,8 @@ class LEDForConditionalGeneration(LEDPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
+ Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
+ config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
(masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
Returns:
@@ -2467,7 +2483,8 @@ class LEDForSequenceClassification(LEDPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
if labels is not None:
diff --git a/src/transformers/models/led/modeling_tf_led.py b/src/transformers/models/led/modeling_tf_led.py
index a0d5dc503d..491d5334b0 100644
--- a/src/transformers/models/led/modeling_tf_led.py
+++ b/src/transformers/models/led/modeling_tf_led.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" TF 2.0 LED model. """
+""" TF 2.0 LED model."""
import random
@@ -617,11 +617,9 @@ class TFLEDEncoderSelfAttention(tf.keras.layers.Layer):
window_overlap = num_rows = 4
```
- (pad & diagonalize) =>
- [ 0.4983, 2.6918, -0.0071, 1.0492, 0.0000, 0.0000, 0.0000
- 0.0000, -1.8348, 0.7672, 0.2986, 0.0285, 0.0000, 0.0000
- 0.0000, 0.0000, -0.7584, 0.4206, -0.0405, 0.1599, 0.0000
- 0.0000, 0.0000, 0.0000, 2.0514, -1.1600, 0.5372, 0.2629 ]
+ (pad & diagonalize) => [ 0.4983, 2.6918, -0.0071, 1.0492, 0.0000, 0.0000, 0.0000
+ 0.0000, -1.8348, 0.7672, 0.2986, 0.0285, 0.0000, 0.0000 0.0000, 0.0000, -0.7584, 0.4206,
+ -0.0405, 0.1599, 0.0000 0.0000, 0.0000, 0.0000, 2.0514, -1.1600, 0.5372, 0.2629 ]
"""
total_num_heads, num_chunks, window_overlap, hidden_dim = shape_list(chunked_hidden_states)
paddings = tf.convert_to_tensor([[0, 0], [0, 0], [0, 0], [0, window_overlap + 1]])
@@ -1209,7 +1207,8 @@ class TFLEDDecoderLayer(tf.keras.layers.Layer):
hidden_states (`tf.Tensor`): input to the layer of shape *(seq_len, batch, embed_dim)*
attention_mask (`tf.Tensor`): attention mask of size
*(batch, 1, tgt_len, src_len)* where padding elements are indicated by very large negative values.
- encoder_hidden_states (`tf.Tensor`): cross attention input to the layer of shape *(seq_len, batch, embed_dim)*
+ encoder_hidden_states (`tf.Tensor`):
+ cross attention input to the layer of shape *(seq_len, batch, embed_dim)*
encoder_attention_mask (`tf.Tensor`): encoder attention mask of size
*(batch, 1, tgt_len, src_len)* where padding elements are indicated by very large negative values.
layer_head_mask (`tf.Tensor`): mask for attention heads in a given layer of size
@@ -1315,26 +1314,28 @@ class TFLEDEncoderBaseModelOutput(ModelOutput):
last_hidden_state (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`):
Sequence of hidden-states at the output of the last layer of the model.
hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x + attention_window + 1)`, where `x` is the number of tokens with global attention mask.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x +
+ attention_window + 1)`, where `x` is the number of tokens with global attention mask.
Local attentions weights after the attention softmax, used to compute the weighted average in the
self-attention heads. Those are the attention weights from every token in the sequence to every token with
- global attention (first `x` values) and to every token in the attention window (remaining
- `attention_window + 1` values). Note that the first `x` values refer to tokens with fixed positions in
- the text, but the remaining `attention_window + 1` values refer to tokens with relative positions: the
- attention weight of a token to itself is located at index `x + attention_window / 2` and the
- `attention_window / 2` preceding (succeeding) values are the attention weights to the `attention_window / 2` preceding (succeeding) tokens. If the attention window contains a token with global attention, the
- attention weight at the corresponding index is set to 0; the value should be accessed from the first `x`
- attention weights. If a token has global attention, the attention weights to all other tokens in
- `attentions` is set to 0, the values should be accessed from `global_attentions`.
+ global attention (first `x` values) and to every token in the attention window (remaining `attention_window
+ + 1` values). Note that the first `x` values refer to tokens with fixed positions in the text, but the
+ remaining `attention_window + 1` values refer to tokens with relative positions: the attention weight of a
+ token to itself is located at index `x + attention_window / 2` and the `attention_window / 2` preceding
+ (succeeding) values are the attention weights to the `attention_window / 2` preceding (succeeding) tokens.
+ If the attention window contains a token with global attention, the attention weight at the corresponding
+ index is set to 0; the value should be accessed from the first `x` attention weights. If a token has global
+ attention, the attention weights to all other tokens in `attentions` is set to 0, the values should be
+ accessed from `global_attentions`.
global_attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x)`,
- where `x` is the number of tokens with global attention mask.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x)`, where `x`
+ is the number of tokens with global attention mask.
Global attentions weights after the attention softmax, used to compute the weighted average in the
self-attention heads. Those are the attention weights from every token with global attention to every token
@@ -1357,42 +1358,47 @@ class TFLEDSeq2SeqModelOutput(ModelOutput):
last_hidden_state (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`):
Sequence of hidden-states at the output of the last layer of the decoder of the model.
- If `past_key_values` is used only the last hidden-state of the sequences of shape `(batch_size, 1, hidden_size)` is output.
+ If `past_key_values` is used only the last hidden-state of the sequences of shape `(batch_size, 1,
+ hidden_size)` is output.
past_key_values (`List[tf.Tensor]`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- List of `tf.Tensor` of length `config.n_layers`, with each tensor of shape `(2, batch_size, num_heads, sequence_length, embed_size_per_head)`).
+ List of `tf.Tensor` of length `config.n_layers`, with each tensor of shape `(2, batch_size, num_heads,
+ sequence_length, embed_size_per_head)`).
Contains pre-computed hidden-states (key and values in the attention blocks) of the decoder that can be
used (see `past_key_values` input) to speed up sequential decoding.
decoder_hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the decoder at the output of each layer plus the initial embedding outputs.
decoder_attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the decoder, after the attention softmax, used to compute the weighted average in the
self-attention heads.
cross_attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the decoder's cross-attention layer, after the attention softmax, used to compute the
weighted average in the cross-attention heads.
encoder_last_hidden_state (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
Sequence of hidden-states at the output of the last layer of the encoder of the model.
encoder_hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the encoder at the output of each layer plus the initial embedding outputs.
encoder_attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the encoder, after the attention softmax, used to compute the weighted average in the
self-attention heads.
encoder_global_attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x)`,
- where `x` is the number of tokens with global attention mask.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x)`, where `x`
+ is the number of tokens with global attention mask.
Global attentions weights after the attention softmax, used to compute the weighted average in the
self-attention heads. Those are the attention weights from every token with global attention to every token
@@ -1421,40 +1427,44 @@ class TFLEDSeq2SeqLMOutput(ModelOutput):
logits (`tf.Tensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
past_key_values (`List[tf.Tensor]`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- List of `tf.Tensor` of length `config.n_layers`, with each tensor of shape `(2, batch_size, num_heads, sequence_length, embed_size_per_head)`).
+ List of `tf.Tensor` of length `config.n_layers`, with each tensor of shape `(2, batch_size, num_heads,
+ sequence_length, embed_size_per_head)`).
Contains pre-computed hidden-states (key and values in the attention blocks) of the decoder that can be
used (see `past_key_values` input) to speed up sequential decoding.
decoder_hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the decoder at the output of each layer plus the initial embedding outputs.
decoder_attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the decoder, after the attention softmax, used to compute the weighted average in the
self-attention heads.
cross_attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the decoder's cross-attention layer, after the attention softmax, used to compute the
weighted average in the cross-attention heads.
encoder_last_hidden_state (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
Sequence of hidden-states at the output of the last layer of the encoder of the model.
encoder_hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the encoder at the output of each layer plus the initial embedding outputs.
encoder_attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the encoder, after the attention softmax, used to compute the weighted average in the
self-attention heads.
encoder_global_attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x)`,
- where `x` is the number of tokens with global attention mask.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x)`, where `x`
+ is the number of tokens with global attention mask.
Global attentions weights after the attention softmax, used to compute the weighted average in the
self-attention heads. Those are the attention weights from every token with global attention to every token
@@ -1474,13 +1484,13 @@ class TFLEDSeq2SeqLMOutput(ModelOutput):
LED_START_DOCSTRING = r"""
- This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading or saving, resizing the input
- embeddings, pruning heads etc.)
+ This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use
- it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage
- and behavior.
+ This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it
+ as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and
+ behavior.
@@ -1489,11 +1499,11 @@ LED_START_DOCSTRING = r"""
- having all inputs as keyword arguments (like PyTorch models), or
- having all inputs as a list, tuple or dict in the first positional arguments.
- This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all
- the tensors in the first argument of the model call function: `model(inputs)`.
+ This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all the
+ tensors in the first argument of the model call function: `model(inputs)`.
- If you choose this second option, there are three possibilities you can use to gather all the input Tensors in
- the first positional argument :
+ If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the
+ first positional argument :
- a single Tensor with `input_ids` only and nothing else: `model(input_ids)`
- a list of varying length with one or several input Tensors IN THE ORDER given in the docstring:
@@ -1506,8 +1516,7 @@ LED_START_DOCSTRING = r"""
Args:
config ([`LEDConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~TFPreTrainedModel.from_pretrained`] method to load the
- model weights.
+ configuration. Check out the [`~TFPreTrainedModel.from_pretrained`] method to load the model weights.
"""
LED_INPUTS_DOCSTRING = r"""
@@ -1515,9 +1524,8 @@ LED_INPUTS_DOCSTRING = r"""
input_ids (`tf.Tensor` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`BertTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`BertTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`tf.Tensor` of shape `({0})`, *optional*):
@@ -1530,15 +1538,13 @@ LED_INPUTS_DOCSTRING = r"""
decoder_input_ids (`tf.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
Indices of decoder input sequence tokens in the vocabulary.
- Indices can be obtained using [`LedTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`LedTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
- LED uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If
- `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
- `past_key_values`).
+ LED uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values`
+ is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`).
decoder_attention_mask (`tf.Tensor` of shape `(batch_size, target_sequence_length)`, *optional*):
will be made by default and ignore pad tokens. It is not recommended to set this for most use cases.
head_mask (`tf.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*):
@@ -1558,12 +1564,12 @@ LED_INPUTS_DOCSTRING = r"""
of shape `(batch_size, sequence_length, hidden_size)` is a sequence of
past_key_values (`Tuple[Tuple[tf.Tensor]]` of length `config.n_layers`)
contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ `decoder_input_ids` of shape `(batch_size, sequence_length)`.
use_cache (`bool`, *optional*, defaults to `True`):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`). Set to `False` during training, `True` during generation
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`). Set to `False` during training, `True` during generation
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the
@@ -1573,8 +1579,8 @@ LED_INPUTS_DOCSTRING = r"""
more detail. This argument can be used only in eager mode, in graph mode the value in the config will be
used instead.
return_dict (`bool`, *optional*):
- Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This
- argument can be used in eager mode, in graph mode the value will always be set to True.
+ Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This argument can be used
+ in eager mode, in graph mode the value will always be set to True.
training (`bool`, *optional*, defaults to `False`):
Whether or not to use the model in training mode (some modules like dropout modules have different
behaviors between training and evaluation).
@@ -1646,9 +1652,8 @@ class TFLEDEncoder(tf.keras.layers.Layer):
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
provide it.
- Indices can be obtained using [`LEDTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`]
- for details.
+ Indices can be obtained using [`LEDTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -1665,9 +1670,9 @@ class TFLEDEncoder(tf.keras.layers.Layer):
- 0 indicates the head is **masked**.
inputs_embeds (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded
- representation. This is useful if you want more control over how to convert `input_ids` indices
- into associated vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
+ This is useful if you want more control over how to convert `input_ids` indices into associated vectors
+ than the model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under
returned tensors for more detail.
@@ -1900,9 +1905,8 @@ class TFLEDDecoder(tf.keras.layers.Layer):
Args:
input_ids (`tf.Tensor` of shape `(batch_size, sequence_length)`):
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
- provide it. Indices can be obtained using [`LEDTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`]
- for details. [What are input IDs?](../glossary#input-ids)
+ provide it. Indices can be obtained using [`LEDTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids)
attention_mask (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
@@ -1935,10 +1939,12 @@ class TFLEDDecoder(tf.keras.layers.Layer):
past_key_values (`Tuple[Tuple[tf.Tensor]]` of length `config.n_layers` with each tuple having 2 tuples each of which has 2 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
Contains precomputed key and value hidden-states of the attention blocks. Can be used to speed up
decoding. If `past_key_values` are used, the user can optionally input only the last
- `decoder_input_ids` (those that don't have their past key value states given to this model) of
- shape `(batch_size, 1)` instead of all ``decoder_input_ids``` of shape `(batch_size,
- sequence_length)`. inputs_embeds (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices
- into associated vectors than the model's internal embedding lookup matrix.
+ `decoder_input_ids` (those that don't have their past key value states given to this model) of shape
+ `(batch_size, 1)` instead of all ``decoder_input_ids``` of shape `(batch_size, sequence_length)`.
+ inputs_embeds (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
+ This is useful if you want more control over how to convert `input_ids` indices into associated vectors
+ than the model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under
returned tensors for more detail.
diff --git a/src/transformers/models/led/tokenization_led.py b/src/transformers/models/led/tokenization_led.py
index eca8098817..fa1f7c8eb3 100644
--- a/src/transformers/models/led/tokenization_led.py
+++ b/src/transformers/models/led/tokenization_led.py
@@ -40,11 +40,10 @@ class LEDTokenizer(BartTokenizer):
"""
Construct a LED tokenizer.
- [`LEDTokenizer`] is identical to [`BartTokenizer`] and runs end-to-end
- tokenization: punctuation splitting and wordpiece.
+ [`LEDTokenizer`] is identical to [`BartTokenizer`] and runs end-to-end tokenization: punctuation splitting and
+ wordpiece.
- Refer to superclass [`BartTokenizer`] for usage examples and documentation concerning
- parameters.
+ Refer to superclass [`BartTokenizer`] for usage examples and documentation concerning parameters.
"""
pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
diff --git a/src/transformers/models/led/tokenization_led_fast.py b/src/transformers/models/led/tokenization_led_fast.py
index b815c806f3..452725b004 100644
--- a/src/transformers/models/led/tokenization_led_fast.py
+++ b/src/transformers/models/led/tokenization_led_fast.py
@@ -41,11 +41,10 @@ class LEDTokenizerFast(BartTokenizerFast):
r"""
Construct a "fast" LED tokenizer (backed by HuggingFace's *tokenizers* library).
- [`LEDTokenizerFast`] is identical to [`BartTokenizerFast`] and runs
- end-to-end tokenization: punctuation splitting and wordpiece.
+ [`LEDTokenizerFast`] is identical to [`BartTokenizerFast`] and runs end-to-end tokenization: punctuation splitting
+ and wordpiece.
- Refer to superclass [`BartTokenizerFast`] for usage examples and documentation concerning
- parameters.
+ Refer to superclass [`BartTokenizerFast`] for usage examples and documentation concerning parameters.
"""
pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
diff --git a/src/transformers/models/longformer/configuration_longformer.py b/src/transformers/models/longformer/configuration_longformer.py
index f10f4a4cd3..59f6fc90bb 100644
--- a/src/transformers/models/longformer/configuration_longformer.py
+++ b/src/transformers/models/longformer/configuration_longformer.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" Longformer configuration """
+""" Longformer configuration"""
from collections import OrderedDict
from typing import List, Mapping, Union
@@ -34,22 +34,21 @@ LONGFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class LongformerConfig(RobertaConfig):
r"""
- This is the configuration class to store the configuration of a [`LongformerModel`] or a
- [`TFLongformerModel`]. It is used to instantiate a Longformer model according to the specified
- arguments, defining the model architecture.
+ This is the configuration class to store the configuration of a [`LongformerModel`] or a [`TFLongformerModel`]. It
+ is used to instantiate a Longformer model according to the specified arguments, defining the model architecture.
- This is the configuration class to store the configuration of a [`LongformerModel`]. It is used
- to instantiate an Longformer model according to the specified arguments, defining the model architecture.
- Instantiating a configuration with the defaults will yield a similar configuration to that of the RoBERTa
+ This is the configuration class to store the configuration of a [`LongformerModel`]. It is used to instantiate an
+ Longformer model according to the specified arguments, defining the model architecture. Instantiating a
+ configuration with the defaults will yield a similar configuration to that of the RoBERTa
[roberta-base](https://huggingface.co/roberta-base) architecture with a sequence length 4,096.
- The [`LongformerConfig`] class directly inherits [`RobertaConfig`]. It reuses
- the same defaults. Please check the parent class for more information.
+ The [`LongformerConfig`] class directly inherits [`RobertaConfig`]. It reuses the same defaults. Please check the
+ parent class for more information.
Args:
attention_window (`int` or `List[int]`, *optional*, defaults to 512):
- Size of an attention window around each token. If an `int`, use the same size for all layers. To
- specify a different window size for each layer, use a `List[int]` where `len(attention_window) == num_hidden_layers`.
+ Size of an attention window around each token. If an `int`, use the same size for all layers. To specify a
+ different window size for each layer, use a `List[int]` where `len(attention_window) == num_hidden_layers`.
Example:
diff --git a/src/transformers/models/longformer/modeling_longformer.py b/src/transformers/models/longformer/modeling_longformer.py
index 056a85450d..987897224c 100755
--- a/src/transformers/models/longformer/modeling_longformer.py
+++ b/src/transformers/models/longformer/modeling_longformer.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-"""PyTorch Longformer model. """
+"""PyTorch Longformer model."""
import math
from dataclasses import dataclass
@@ -66,26 +66,28 @@ class LongformerBaseModelOutput(ModelOutput):
last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
Sequence of hidden-states at the output of the last layer of the model.
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x + attention_window + 1)`, where `x` is the number of tokens with global attention
- mask.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x +
+ attention_window + 1)`, where `x` is the number of tokens with global attention mask.
Local attentions weights after the attention softmax, used to compute the weighted average in the
self-attention heads. Those are the attention weights from every token in the sequence to every token with
- global attention (first `x` values) and to every token in the attention window (remaining
- `attention_window + 1` values). Note that the first `x` values refer to tokens with fixed positions in
- the text, but the remaining `attention_window + 1` values refer to tokens with relative positions: the
- attention weight of a token to itself is located at index `x + attention_window / 2` and the
- `attention_window / 2` preceding (succeeding) values are the attention weights to the `attention_window / 2` preceding (succeeding) tokens. If the attention window contains a token with global attention, the
- attention weight at the corresponding index is set to 0; the value should be accessed from the first `x`
- attention weights. If a token has global attention, the attention weights to all other tokens in
- `attentions` is set to 0, the values should be accessed from `global_attentions`.
+ global attention (first `x` values) and to every token in the attention window (remaining `attention_window
+ + 1` values). Note that the first `x` values refer to tokens with fixed positions in the text, but the
+ remaining `attention_window + 1` values refer to tokens with relative positions: the attention weight of a
+ token to itself is located at index `x + attention_window / 2` and the `attention_window / 2` preceding
+ (succeeding) values are the attention weights to the `attention_window / 2` preceding (succeeding) tokens.
+ If the attention window contains a token with global attention, the attention weight at the corresponding
+ index is set to 0; the value should be accessed from the first `x` attention weights. If a token has global
+ attention, the attention weights to all other tokens in `attentions` is set to 0, the values should be
+ accessed from `global_attentions`.
global_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x)`, where `x` is the number of tokens with global attention mask.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x)`,
+ where `x` is the number of tokens with global attention mask.
Global attentions weights after the attention softmax, used to compute the weighted average in the
self-attention heads. Those are the attention weights from every token with global attention to every token
@@ -111,26 +113,28 @@ class LongformerBaseModelOutputWithPooling(ModelOutput):
Linear layer and a Tanh activation function. The Linear layer weights are trained from the next sentence
prediction (classification) objective during pretraining.
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x + attention_window + 1)`, where `x` is the number of tokens with global attention
- mask.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x +
+ attention_window + 1)`, where `x` is the number of tokens with global attention mask.
Local attentions weights after the attention softmax, used to compute the weighted average in the
self-attention heads. Those are the attention weights from every token in the sequence to every token with
- global attention (first `x` values) and to every token in the attention window (remaining
- `attention_window + 1` values). Note that the first `x` values refer to tokens with fixed positions in
- the text, but the remaining `attention_window + 1` values refer to tokens with relative positions: the
- attention weight of a token to itself is located at index `x + attention_window / 2` and the
- `attention_window / 2` preceding (succeeding) values are the attention weights to the `attention_window / 2` preceding (succeeding) tokens. If the attention window contains a token with global attention, the
- attention weight at the corresponding index is set to 0; the value should be accessed from the first `x`
- attention weights. If a token has global attention, the attention weights to all other tokens in
- `attentions` is set to 0, the values should be accessed from `global_attentions`.
+ global attention (first `x` values) and to every token in the attention window (remaining `attention_window
+ + 1` values). Note that the first `x` values refer to tokens with fixed positions in the text, but the
+ remaining `attention_window + 1` values refer to tokens with relative positions: the attention weight of a
+ token to itself is located at index `x + attention_window / 2` and the `attention_window / 2` preceding
+ (succeeding) values are the attention weights to the `attention_window / 2` preceding (succeeding) tokens.
+ If the attention window contains a token with global attention, the attention weight at the corresponding
+ index is set to 0; the value should be accessed from the first `x` attention weights. If a token has global
+ attention, the attention weights to all other tokens in `attentions` is set to 0, the values should be
+ accessed from `global_attentions`.
global_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x)`, where `x` is the number of tokens with global attention mask.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x)`,
+ where `x` is the number of tokens with global attention mask.
Global attentions weights after the attention softmax, used to compute the weighted average in the
self-attention heads. Those are the attention weights from every token with global attention to every token
@@ -155,26 +159,28 @@ class LongformerMaskedLMOutput(ModelOutput):
logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x + attention_window + 1)`, where `x` is the number of tokens with global attention
- mask.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x +
+ attention_window + 1)`, where `x` is the number of tokens with global attention mask.
Local attentions weights after the attention softmax, used to compute the weighted average in the
self-attention heads. Those are the attention weights from every token in the sequence to every token with
- global attention (first `x` values) and to every token in the attention window (remaining
- `attention_window + 1` values). Note that the first `x` values refer to tokens with fixed positions in
- the text, but the remaining `attention_window + 1` values refer to tokens with relative positions: the
- attention weight of a token to itself is located at index `x + attention_window / 2` and the
- `attention_window / 2` preceding (succeeding) values are the attention weights to the `attention_window / 2` preceding (succeeding) tokens. If the attention window contains a token with global attention, the
- attention weight at the corresponding index is set to 0; the value should be accessed from the first `x`
- attention weights. If a token has global attention, the attention weights to all other tokens in
- `attentions` is set to 0, the values should be accessed from `global_attentions`.
+ global attention (first `x` values) and to every token in the attention window (remaining `attention_window
+ + 1` values). Note that the first `x` values refer to tokens with fixed positions in the text, but the
+ remaining `attention_window + 1` values refer to tokens with relative positions: the attention weight of a
+ token to itself is located at index `x + attention_window / 2` and the `attention_window / 2` preceding
+ (succeeding) values are the attention weights to the `attention_window / 2` preceding (succeeding) tokens.
+ If the attention window contains a token with global attention, the attention weight at the corresponding
+ index is set to 0; the value should be accessed from the first `x` attention weights. If a token has global
+ attention, the attention weights to all other tokens in `attentions` is set to 0, the values should be
+ accessed from `global_attentions`.
global_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x)`, where `x` is the number of tokens with global attention mask.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x)`,
+ where `x` is the number of tokens with global attention mask.
Global attentions weights after the attention softmax, used to compute the weighted average in the
self-attention heads. Those are the attention weights from every token with global attention to every token
@@ -201,26 +207,28 @@ class LongformerQuestionAnsweringModelOutput(ModelOutput):
end_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length)`):
Span-end scores (before SoftMax).
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x + attention_window + 1)`, where `x` is the number of tokens with global attention
- mask.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x +
+ attention_window + 1)`, where `x` is the number of tokens with global attention mask.
Local attentions weights after the attention softmax, used to compute the weighted average in the
self-attention heads. Those are the attention weights from every token in the sequence to every token with
- global attention (first `x` values) and to every token in the attention window (remaining
- `attention_window + 1` values). Note that the first `x` values refer to tokens with fixed positions in
- the text, but the remaining `attention_window + 1` values refer to tokens with relative positions: the
- attention weight of a token to itself is located at index `x + attention_window / 2` and the
- `attention_window / 2` preceding (succeeding) values are the attention weights to the `attention_window / 2` preceding (succeeding) tokens. If the attention window contains a token with global attention, the
- attention weight at the corresponding index is set to 0; the value should be accessed from the first `x`
- attention weights. If a token has global attention, the attention weights to all other tokens in
- `attentions` is set to 0, the values should be accessed from `global_attentions`.
+ global attention (first `x` values) and to every token in the attention window (remaining `attention_window
+ + 1` values). Note that the first `x` values refer to tokens with fixed positions in the text, but the
+ remaining `attention_window + 1` values refer to tokens with relative positions: the attention weight of a
+ token to itself is located at index `x + attention_window / 2` and the `attention_window / 2` preceding
+ (succeeding) values are the attention weights to the `attention_window / 2` preceding (succeeding) tokens.
+ If the attention window contains a token with global attention, the attention weight at the corresponding
+ index is set to 0; the value should be accessed from the first `x` attention weights. If a token has global
+ attention, the attention weights to all other tokens in `attentions` is set to 0, the values should be
+ accessed from `global_attentions`.
global_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x)`, where `x` is the number of tokens with global attention mask.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x)`,
+ where `x` is the number of tokens with global attention mask.
Global attentions weights after the attention softmax, used to compute the weighted average in the
self-attention heads. Those are the attention weights from every token with global attention to every token
@@ -246,26 +254,28 @@ class LongformerSequenceClassifierOutput(ModelOutput):
logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
Classification (or regression if config.num_labels==1) scores (before SoftMax).
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x + attention_window + 1)`, where `x` is the number of tokens with global attention
- mask.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x +
+ attention_window + 1)`, where `x` is the number of tokens with global attention mask.
Local attentions weights after the attention softmax, used to compute the weighted average in the
self-attention heads. Those are the attention weights from every token in the sequence to every token with
- global attention (first `x` values) and to every token in the attention window (remaining
- `attention_window + 1` values). Note that the first `x` values refer to tokens with fixed positions in
- the text, but the remaining `attention_window + 1` values refer to tokens with relative positions: the
- attention weight of a token to itself is located at index `x + attention_window / 2` and the
- `attention_window / 2` preceding (succeeding) values are the attention weights to the `attention_window / 2` preceding (succeeding) tokens. If the attention window contains a token with global attention, the
- attention weight at the corresponding index is set to 0; the value should be accessed from the first `x`
- attention weights. If a token has global attention, the attention weights to all other tokens in
- `attentions` is set to 0, the values should be accessed from `global_attentions`.
+ global attention (first `x` values) and to every token in the attention window (remaining `attention_window
+ + 1` values). Note that the first `x` values refer to tokens with fixed positions in the text, but the
+ remaining `attention_window + 1` values refer to tokens with relative positions: the attention weight of a
+ token to itself is located at index `x + attention_window / 2` and the `attention_window / 2` preceding
+ (succeeding) values are the attention weights to the `attention_window / 2` preceding (succeeding) tokens.
+ If the attention window contains a token with global attention, the attention weight at the corresponding
+ index is set to 0; the value should be accessed from the first `x` attention weights. If a token has global
+ attention, the attention weights to all other tokens in `attentions` is set to 0, the values should be
+ accessed from `global_attentions`.
global_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x)`, where `x` is the number of tokens with global attention mask.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x)`,
+ where `x` is the number of tokens with global attention mask.
Global attentions weights after the attention softmax, used to compute the weighted average in the
self-attention heads. Those are the attention weights from every token with global attention to every token
@@ -292,26 +302,28 @@ class LongformerMultipleChoiceModelOutput(ModelOutput):
Classification scores (before SoftMax).
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x + attention_window + 1)`, where `x` is the number of tokens with global attention
- mask.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x +
+ attention_window + 1)`, where `x` is the number of tokens with global attention mask.
Local attentions weights after the attention softmax, used to compute the weighted average in the
self-attention heads. Those are the attention weights from every token in the sequence to every token with
- global attention (first `x` values) and to every token in the attention window (remaining
- `attention_window + 1` values). Note that the first `x` values refer to tokens with fixed positions in
- the text, but the remaining `attention_window + 1` values refer to tokens with relative positions: the
- attention weight of a token to itself is located at index `x + attention_window / 2` and the
- `attention_window / 2` preceding (succeeding) values are the attention weights to the `attention_window / 2` preceding (succeeding) tokens. If the attention window contains a token with global attention, the
- attention weight at the corresponding index is set to 0; the value should be accessed from the first `x`
- attention weights. If a token has global attention, the attention weights to all other tokens in
- `attentions` is set to 0, the values should be accessed from `global_attentions`.
+ global attention (first `x` values) and to every token in the attention window (remaining `attention_window
+ + 1` values). Note that the first `x` values refer to tokens with fixed positions in the text, but the
+ remaining `attention_window + 1` values refer to tokens with relative positions: the attention weight of a
+ token to itself is located at index `x + attention_window / 2` and the `attention_window / 2` preceding
+ (succeeding) values are the attention weights to the `attention_window / 2` preceding (succeeding) tokens.
+ If the attention window contains a token with global attention, the attention weight at the corresponding
+ index is set to 0; the value should be accessed from the first `x` attention weights. If a token has global
+ attention, the attention weights to all other tokens in `attentions` is set to 0, the values should be
+ accessed from `global_attentions`.
global_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x)`, where `x` is the number of tokens with global attention mask.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x)`,
+ where `x` is the number of tokens with global attention mask.
Global attentions weights after the attention softmax, used to compute the weighted average in the
self-attention heads. Those are the attention weights from every token with global attention to every token
@@ -336,26 +348,28 @@ class LongformerTokenClassifierOutput(ModelOutput):
logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.num_labels)`):
Classification scores (before SoftMax).
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x + attention_window + 1)`, where `x` is the number of tokens with global attention
- mask.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x +
+ attention_window + 1)`, where `x` is the number of tokens with global attention mask.
Local attentions weights after the attention softmax, used to compute the weighted average in the
self-attention heads. Those are the attention weights from every token in the sequence to every token with
- global attention (first `x` values) and to every token in the attention window (remaining
- `attention_window + 1` values). Note that the first `x` values refer to tokens with fixed positions in
- the text, but the remaining `attention_window + 1` values refer to tokens with relative positions: the
- attention weight of a token to itself is located at index `x + attention_window / 2` and the
- `attention_window / 2` preceding (succeeding) values are the attention weights to the `attention_window / 2` preceding (succeeding) tokens. If the attention window contains a token with global attention, the
- attention weight at the corresponding index is set to 0; the value should be accessed from the first `x`
- attention weights. If a token has global attention, the attention weights to all other tokens in
- `attentions` is set to 0, the values should be accessed from `global_attentions`.
+ global attention (first `x` values) and to every token in the attention window (remaining `attention_window
+ + 1` values). Note that the first `x` values refer to tokens with fixed positions in the text, but the
+ remaining `attention_window + 1` values refer to tokens with relative positions: the attention weight of a
+ token to itself is located at index `x + attention_window / 2` and the `attention_window / 2` preceding
+ (succeeding) values are the attention weights to the `attention_window / 2` preceding (succeeding) tokens.
+ If the attention window contains a token with global attention, the attention weight at the corresponding
+ index is set to 0; the value should be accessed from the first `x` attention weights. If a token has global
+ attention, the attention weights to all other tokens in `attentions` is set to 0, the values should be
+ accessed from `global_attentions`.
global_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x)`, where `x` is the number of tokens with global attention mask.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x)`,
+ where `x` is the number of tokens with global attention mask.
Global attentions weights after the attention softmax, used to compute the weighted average in the
self-attention heads. Those are the attention weights from every token with global attention to every token
@@ -719,11 +733,9 @@ class LongformerSelfAttention(nn.Module):
window_overlap = num_rows = 4
```
- (pad & diagonalize) =>
- [ 0.4983, 2.6918, -0.0071, 1.0492, 0.0000, 0.0000, 0.0000
- 0.0000, -1.8348, 0.7672, 0.2986, 0.0285, 0.0000, 0.0000
- 0.0000, 0.0000, -0.7584, 0.4206, -0.0405, 0.1599, 0.0000
- 0.0000, 0.0000, 0.0000, 2.0514, -1.1600, 0.5372, 0.2629 ]
+ (pad & diagonalize) => [ 0.4983, 2.6918, -0.0071, 1.0492, 0.0000, 0.0000, 0.0000
+ 0.0000, -1.8348, 0.7672, 0.2986, 0.0285, 0.0000, 0.0000 0.0000, 0.0000, -0.7584, 0.4206,
+ -0.0405, 0.1599, 0.0000 0.0000, 0.0000, 0.0000, 2.0514, -1.1600, 0.5372, 0.2629 ]
"""
total_num_heads, num_chunks, window_overlap, hidden_dim = chunked_hidden_states.size()
chunked_hidden_states = nn.functional.pad(
@@ -1372,19 +1384,18 @@ class LongformerPreTrainedModel(PreTrainedModel):
LONGFORMER_START_DOCSTRING = r"""
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
Parameters:
config ([`LongformerConfig`]): Model configuration class with all the parameters of the
model. Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
LONGFORMER_INPUTS_DOCSTRING = r"""
@@ -1392,9 +1403,8 @@ LONGFORMER_INPUTS_DOCSTRING = r"""
input_ids (`torch.LongTensor` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`LongformerTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`LongformerTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
@@ -1428,20 +1438,22 @@ LONGFORMER_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -1459,17 +1471,17 @@ LONGFORMER_INPUTS_DOCSTRING = r"""
)
class LongformerModel(LongformerPreTrainedModel):
"""
- This class copied code from [`RobertaModel`] and overwrote standard self-attention with
- longformer self-attention to provide the ability to process long sequences following the self-attention approach
- described in [Longformer: the Long-Document Transformer](https://arxiv.org/abs/2004.05150) by Iz Beltagy,
- Matthew E. Peters, and Arman Cohan. Longformer self-attention combines a local (sliding window) and global
- attention to extend to long documents without the O(n^2) increase in memory and compute.
+ This class copied code from [`RobertaModel`] and overwrote standard self-attention with longformer self-attention
+ to provide the ability to process long sequences following the self-attention approach described in [Longformer:
+ the Long-Document Transformer](https://arxiv.org/abs/2004.05150) by Iz Beltagy, Matthew E. Peters, and Arman Cohan.
+ Longformer self-attention combines a local (sliding window) and global attention to extend to long documents
+ without the O(n^2) increase in memory and compute.
- The self-attention module `LongformerSelfAttention` implemented here supports the combination of local and
- global attention but it lacks support for autoregressive attention and dilated attention. Autoregressive and
- dilated attention are more relevant for autoregressive language modeling than finetuning on downstream tasks.
- Future release will add support for autoregressive attention, but the support for dilated attention requires a
- custom CUDA kernel to be memory and compute efficient.
+ The self-attention module `LongformerSelfAttention` implemented here supports the combination of local and global
+ attention but it lacks support for autoregressive attention and dilated attention. Autoregressive and dilated
+ attention are more relevant for autoregressive language modeling than finetuning on downstream tasks. Future
+ release will add support for autoregressive attention, but the support for dilated attention requires a custom CUDA
+ kernel to be memory and compute efficient.
"""
@@ -1684,7 +1696,7 @@ class LongformerModel(LongformerPreTrainedModel):
)
-@add_start_docstrings("""Longformer Model with a `language modeling` head on top. """, LONGFORMER_START_DOCSTRING)
+@add_start_docstrings("""Longformer Model with a `language modeling` head on top.""", LONGFORMER_START_DOCSTRING)
class LongformerForMaskedLM(LongformerPreTrainedModel):
_keys_to_ignore_on_load_unexpected = [r"pooler"]
@@ -1722,8 +1734,9 @@ class LongformerForMaskedLM(LongformerPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
kwargs (`Dict[str, any]`, optional, defaults to *{}*):
Used to hide legacy arguments that have been deprecated.
@@ -1827,8 +1840,9 @@ class LongformerForSequenceClassification(LongformerPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1949,12 +1963,12 @@ class LongformerForQuestionAnswering(LongformerPreTrainedModel):
r"""
start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the start of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the end of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
Returns:
@@ -2182,7 +2196,8 @@ class LongformerForMultipleChoice(LongformerPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
+ Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
+ num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
`input_ids` above)
"""
num_choices = input_ids.shape[1] if input_ids is not None else inputs_embeds.shape[1]
diff --git a/src/transformers/models/longformer/modeling_tf_longformer.py b/src/transformers/models/longformer/modeling_tf_longformer.py
index 0137d545da..c47955201d 100644
--- a/src/transformers/models/longformer/modeling_tf_longformer.py
+++ b/src/transformers/models/longformer/modeling_tf_longformer.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-"""Tensorflow Longformer model. """
+"""Tensorflow Longformer model."""
import warnings
from dataclasses import dataclass
@@ -69,26 +69,28 @@ class TFLongformerBaseModelOutput(ModelOutput):
last_hidden_state (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`):
Sequence of hidden-states at the output of the last layer of the model.
hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x + attention_window + 1)`, where `x` is the number of tokens with global attention mask.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x +
+ attention_window + 1)`, where `x` is the number of tokens with global attention mask.
Local attentions weights after the attention softmax, used to compute the weighted average in the
self-attention heads. Those are the attention weights from every token in the sequence to every token with
- global attention (first `x` values) and to every token in the attention window (remaining
- `attention_window + 1` values). Note that the first `x` values refer to tokens with fixed positions in
- the text, but the remaining `attention_window + 1` values refer to tokens with relative positions: the
- attention weight of a token to itself is located at index `x + attention_window / 2` and the
- `attention_window / 2` preceding (succeeding) values are the attention weights to the `attention_window / 2` preceding (succeeding) tokens. If the attention window contains a token with global attention, the
- attention weight at the corresponding index is set to 0; the value should be accessed from the first `x`
- attention weights. If a token has global attention, the attention weights to all other tokens in
- `attentions` is set to 0, the values should be accessed from `global_attentions`.
+ global attention (first `x` values) and to every token in the attention window (remaining `attention_window
+ + 1` values). Note that the first `x` values refer to tokens with fixed positions in the text, but the
+ remaining `attention_window + 1` values refer to tokens with relative positions: the attention weight of a
+ token to itself is located at index `x + attention_window / 2` and the `attention_window / 2` preceding
+ (succeeding) values are the attention weights to the `attention_window / 2` preceding (succeeding) tokens.
+ If the attention window contains a token with global attention, the attention weight at the corresponding
+ index is set to 0; the value should be accessed from the first `x` attention weights. If a token has global
+ attention, the attention weights to all other tokens in `attentions` is set to 0, the values should be
+ accessed from `global_attentions`.
global_attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x)`,
- where `x` is the number of tokens with global attention mask.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x)`, where `x`
+ is the number of tokens with global attention mask.
Global attentions weights after the attention softmax, used to compute the weighted average in the
self-attention heads. Those are the attention weights from every token with global attention to every token
@@ -114,26 +116,28 @@ class TFLongformerBaseModelOutputWithPooling(ModelOutput):
Linear layer and a Tanh activation function. The Linear layer weights are trained from the next sentence
prediction (classification) objective during pretraining.
hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x + attention_window + 1)`, where `x` is the number of tokens with global attention mask.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x +
+ attention_window + 1)`, where `x` is the number of tokens with global attention mask.
Local attentions weights after the attention softmax, used to compute the weighted average in the
self-attention heads. Those are the attention weights from every token in the sequence to every token with
- global attention (first `x` values) and to every token in the attention window (remaining
- `attention_window + 1` values). Note that the first `x` values refer to tokens with fixed positions in
- the text, but the remaining `attention_window + 1` values refer to tokens with relative positions: the
- attention weight of a token to itself is located at index `x + attention_window / 2` and the
- `attention_window / 2` preceding (succeeding) values are the attention weights to the `attention_window / 2` preceding (succeeding) tokens. If the attention window contains a token with global attention, the
- attention weight at the corresponding index is set to 0; the value should be accessed from the first `x`
- attention weights. If a token has global attention, the attention weights to all other tokens in
- `attentions` is set to 0, the values should be accessed from `global_attentions`.
+ global attention (first `x` values) and to every token in the attention window (remaining `attention_window
+ + 1` values). Note that the first `x` values refer to tokens with fixed positions in the text, but the
+ remaining `attention_window + 1` values refer to tokens with relative positions: the attention weight of a
+ token to itself is located at index `x + attention_window / 2` and the `attention_window / 2` preceding
+ (succeeding) values are the attention weights to the `attention_window / 2` preceding (succeeding) tokens.
+ If the attention window contains a token with global attention, the attention weight at the corresponding
+ index is set to 0; the value should be accessed from the first `x` attention weights. If a token has global
+ attention, the attention weights to all other tokens in `attentions` is set to 0, the values should be
+ accessed from `global_attentions`.
global_attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x)`,
- where `x` is the number of tokens with global attention mask.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x)`, where `x`
+ is the number of tokens with global attention mask.
Global attentions weights after the attention softmax, used to compute the weighted average in the
self-attention heads. Those are the attention weights from every token with global attention to every token
@@ -158,26 +162,28 @@ class TFLongformerMaskedLMOutput(ModelOutput):
logits (`tf.Tensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x + attention_window + 1)`, where `x` is the number of tokens with global attention mask.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x +
+ attention_window + 1)`, where `x` is the number of tokens with global attention mask.
Local attentions weights after the attention softmax, used to compute the weighted average in the
self-attention heads. Those are the attention weights from every token in the sequence to every token with
- global attention (first `x` values) and to every token in the attention window (remaining
- `attention_window + 1` values). Note that the first `x` values refer to tokens with fixed positions in
- the text, but the remaining `attention_window + 1` values refer to tokens with relative positions: the
- attention weight of a token to itself is located at index `x + attention_window / 2` and the
- `attention_window / 2` preceding (succeeding) values are the attention weights to the `attention_window / 2` preceding (succeeding) tokens. If the attention window contains a token with global attention, the
- attention weight at the corresponding index is set to 0; the value should be accessed from the first `x`
- attention weights. If a token has global attention, the attention weights to all other tokens in
- `attentions` is set to 0, the values should be accessed from `global_attentions`.
+ global attention (first `x` values) and to every token in the attention window (remaining `attention_window
+ + 1` values). Note that the first `x` values refer to tokens with fixed positions in the text, but the
+ remaining `attention_window + 1` values refer to tokens with relative positions: the attention weight of a
+ token to itself is located at index `x + attention_window / 2` and the `attention_window / 2` preceding
+ (succeeding) values are the attention weights to the `attention_window / 2` preceding (succeeding) tokens.
+ If the attention window contains a token with global attention, the attention weight at the corresponding
+ index is set to 0; the value should be accessed from the first `x` attention weights. If a token has global
+ attention, the attention weights to all other tokens in `attentions` is set to 0, the values should be
+ accessed from `global_attentions`.
global_attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x)`,
- where `x` is the number of tokens with global attention mask.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x)`, where `x`
+ is the number of tokens with global attention mask.
Global attentions weights after the attention softmax, used to compute the weighted average in the
self-attention heads. Those are the attention weights from every token with global attention to every token
@@ -204,26 +210,28 @@ class TFLongformerQuestionAnsweringModelOutput(ModelOutput):
end_logits (`tf.Tensor` of shape `(batch_size, sequence_length)`):
Span-end scores (before SoftMax).
hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x + attention_window + 1)`, where `x` is the number of tokens with global attention mask.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x +
+ attention_window + 1)`, where `x` is the number of tokens with global attention mask.
Local attentions weights after the attention softmax, used to compute the weighted average in the
self-attention heads. Those are the attention weights from every token in the sequence to every token with
- global attention (first `x` values) and to every token in the attention window (remaining
- `attention_window + 1` values). Note that the first `x` values refer to tokens with fixed positions in
- the text, but the remaining `attention_window + 1` values refer to tokens with relative positions: the
- attention weight of a token to itself is located at index `x + attention_window / 2` and the
- `attention_window / 2` preceding (succeeding) values are the attention weights to the `attention_window / 2` preceding (succeeding) tokens. If the attention window contains a token with global attention, the
- attention weight at the corresponding index is set to 0; the value should be accessed from the first `x`
- attention weights. If a token has global attention, the attention weights to all other tokens in
- `attentions` is set to 0, the values should be accessed from `global_attentions`.
+ global attention (first `x` values) and to every token in the attention window (remaining `attention_window
+ + 1` values). Note that the first `x` values refer to tokens with fixed positions in the text, but the
+ remaining `attention_window + 1` values refer to tokens with relative positions: the attention weight of a
+ token to itself is located at index `x + attention_window / 2` and the `attention_window / 2` preceding
+ (succeeding) values are the attention weights to the `attention_window / 2` preceding (succeeding) tokens.
+ If the attention window contains a token with global attention, the attention weight at the corresponding
+ index is set to 0; the value should be accessed from the first `x` attention weights. If a token has global
+ attention, the attention weights to all other tokens in `attentions` is set to 0, the values should be
+ accessed from `global_attentions`.
global_attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x)`,
- where `x` is the number of tokens with global attention mask.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x)`, where `x`
+ is the number of tokens with global attention mask.
Global attentions weights after the attention softmax, used to compute the weighted average in the
self-attention heads. Those are the attention weights from every token with global attention to every token
@@ -249,26 +257,28 @@ class TFLongformerSequenceClassifierOutput(ModelOutput):
logits (`tf.Tensor` of shape `(batch_size, config.num_labels)`):
Classification (or regression if config.num_labels==1) scores (before SoftMax).
hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x + attention_window + 1)`, where `x` is the number of tokens with global attention mask.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x +
+ attention_window + 1)`, where `x` is the number of tokens with global attention mask.
Local attentions weights after the attention softmax, used to compute the weighted average in the
self-attention heads. Those are the attention weights from every token in the sequence to every token with
- global attention (first `x` values) and to every token in the attention window (remaining
- `attention_window + 1` values). Note that the first `x` values refer to tokens with fixed positions in
- the text, but the remaining `attention_window + 1` values refer to tokens with relative positions: the
- attention weight of a token to itself is located at index `x + attention_window / 2` and the
- `attention_window / 2` preceding (succeeding) values are the attention weights to the `attention_window / 2` preceding (succeeding) tokens. If the attention window contains a token with global attention, the
- attention weight at the corresponding index is set to 0; the value should be accessed from the first `x`
- attention weights. If a token has global attention, the attention weights to all other tokens in
- `attentions` is set to 0, the values should be accessed from `global_attentions`.
+ global attention (first `x` values) and to every token in the attention window (remaining `attention_window
+ + 1` values). Note that the first `x` values refer to tokens with fixed positions in the text, but the
+ remaining `attention_window + 1` values refer to tokens with relative positions: the attention weight of a
+ token to itself is located at index `x + attention_window / 2` and the `attention_window / 2` preceding
+ (succeeding) values are the attention weights to the `attention_window / 2` preceding (succeeding) tokens.
+ If the attention window contains a token with global attention, the attention weight at the corresponding
+ index is set to 0; the value should be accessed from the first `x` attention weights. If a token has global
+ attention, the attention weights to all other tokens in `attentions` is set to 0, the values should be
+ accessed from `global_attentions`.
global_attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x)`,
- where `x` is the number of tokens with global attention mask.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x)`, where `x`
+ is the number of tokens with global attention mask.
Global attentions weights after the attention softmax, used to compute the weighted average in the
self-attention heads. Those are the attention weights from every token with global attention to every token
@@ -295,26 +305,28 @@ class TFLongformerMultipleChoiceModelOutput(ModelOutput):
Classification scores (before SoftMax).
hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x + attention_window + 1)`, where `x` is the number of tokens with global attention mask.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x +
+ attention_window + 1)`, where `x` is the number of tokens with global attention mask.
Local attentions weights after the attention softmax, used to compute the weighted average in the
self-attention heads. Those are the attention weights from every token in the sequence to every token with
- global attention (first `x` values) and to every token in the attention window (remaining
- `attention_window + 1` values). Note that the first `x` values refer to tokens with fixed positions in
- the text, but the remaining `attention_window + 1` values refer to tokens with relative positions: the
- attention weight of a token to itself is located at index `x + attention_window / 2` and the
- `attention_window / 2` preceding (succeeding) values are the attention weights to the `attention_window / 2` preceding (succeeding) tokens. If the attention window contains a token with global attention, the
- attention weight at the corresponding index is set to 0; the value should be accessed from the first `x`
- attention weights. If a token has global attention, the attention weights to all other tokens in
- `attentions` is set to 0, the values should be accessed from `global_attentions`.
+ global attention (first `x` values) and to every token in the attention window (remaining `attention_window
+ + 1` values). Note that the first `x` values refer to tokens with fixed positions in the text, but the
+ remaining `attention_window + 1` values refer to tokens with relative positions: the attention weight of a
+ token to itself is located at index `x + attention_window / 2` and the `attention_window / 2` preceding
+ (succeeding) values are the attention weights to the `attention_window / 2` preceding (succeeding) tokens.
+ If the attention window contains a token with global attention, the attention weight at the corresponding
+ index is set to 0; the value should be accessed from the first `x` attention weights. If a token has global
+ attention, the attention weights to all other tokens in `attentions` is set to 0, the values should be
+ accessed from `global_attentions`.
global_attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x)`,
- where `x` is the number of tokens with global attention mask.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x)`, where `x`
+ is the number of tokens with global attention mask.
Global attentions weights after the attention softmax, used to compute the weighted average in the
self-attention heads. Those are the attention weights from every token with global attention to every token
@@ -339,26 +351,28 @@ class TFLongformerTokenClassifierOutput(ModelOutput):
logits (`tf.Tensor` of shape `(batch_size, sequence_length, config.num_labels)`):
Classification scores (before SoftMax).
hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x + attention_window + 1)`, where `x` is the number of tokens with global attention mask.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x +
+ attention_window + 1)`, where `x` is the number of tokens with global attention mask.
Local attentions weights after the attention softmax, used to compute the weighted average in the
self-attention heads. Those are the attention weights from every token in the sequence to every token with
- global attention (first `x` values) and to every token in the attention window (remaining
- `attention_window + 1` values). Note that the first `x` values refer to tokens with fixed positions in
- the text, but the remaining `attention_window + 1` values refer to tokens with relative positions: the
- attention weight of a token to itself is located at index `x + attention_window / 2` and the
- `attention_window / 2` preceding (succeeding) values are the attention weights to the `attention_window / 2` preceding (succeeding) tokens. If the attention window contains a token with global attention, the
- attention weight at the corresponding index is set to 0; the value should be accessed from the first `x`
- attention weights. If a token has global attention, the attention weights to all other tokens in
- `attentions` is set to 0, the values should be accessed from `global_attentions`.
+ global attention (first `x` values) and to every token in the attention window (remaining `attention_window
+ + 1` values). Note that the first `x` values refer to tokens with fixed positions in the text, but the
+ remaining `attention_window + 1` values refer to tokens with relative positions: the attention weight of a
+ token to itself is located at index `x + attention_window / 2` and the `attention_window / 2` preceding
+ (succeeding) values are the attention weights to the `attention_window / 2` preceding (succeeding) tokens.
+ If the attention window contains a token with global attention, the attention weight at the corresponding
+ index is set to 0; the value should be accessed from the first `x` attention weights. If a token has global
+ attention, the attention weights to all other tokens in `attentions` is set to 0, the values should be
+ accessed from `global_attentions`.
global_attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x)`,
- where `x` is the number of tokens with global attention mask.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, x)`, where `x`
+ is the number of tokens with global attention mask.
Global attentions weights after the attention softmax, used to compute the weighted average in the
self-attention heads. Those are the attention weights from every token with global attention to every token
@@ -1131,11 +1145,9 @@ class TFLongformerSelfAttention(tf.keras.layers.Layer):
window_overlap = num_rows = 4
```
- (pad & diagonalize) =>
- [ 0.4983, 2.6918, -0.0071, 1.0492, 0.0000, 0.0000, 0.0000
- 0.0000, -1.8348, 0.7672, 0.2986, 0.0285, 0.0000, 0.0000
- 0.0000, 0.0000, -0.7584, 0.4206, -0.0405, 0.1599, 0.0000
- 0.0000, 0.0000, 0.0000, 2.0514, -1.1600, 0.5372, 0.2629 ]
+ (pad & diagonalize) => [ 0.4983, 2.6918, -0.0071, 1.0492, 0.0000, 0.0000, 0.0000
+ 0.0000, -1.8348, 0.7672, 0.2986, 0.0285, 0.0000, 0.0000 0.0000, 0.0000, -0.7584, 0.4206,
+ -0.0405, 0.1599, 0.0000 0.0000, 0.0000, 0.0000, 2.0514, -1.1600, 0.5372, 0.2629 ]
"""
total_num_heads, num_chunks, window_overlap, hidden_dim = shape_list(chunked_hidden_states)
paddings = tf.convert_to_tensor([[0, 0], [0, 0], [0, 0], [0, window_overlap + 1]])
@@ -1861,13 +1873,13 @@ class TFLongformerPreTrainedModel(TFPreTrainedModel):
LONGFORMER_START_DOCSTRING = r"""
- This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading or saving, resizing the input
- embeddings, pruning heads etc.)
+ This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use
- it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage
- and behavior.
+ This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it
+ as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and
+ behavior.
@@ -1876,11 +1888,11 @@ LONGFORMER_START_DOCSTRING = r"""
- having all inputs as keyword arguments (like PyTorch models), or
- having all inputs as a list, tuple or dict in the first positional arguments.
- This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all
- the tensors in the first argument of the model call function: `model(inputs)`.
+ This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all the
+ tensors in the first argument of the model call function: `model(inputs)`.
- If you choose this second option, there are three possibilities you can use to gather all the input Tensors in
- the first positional argument :
+ If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the
+ first positional argument :
- a single Tensor with `input_ids` only and nothing else: `model(inputs_ids)`
- a list of varying length with one or several input Tensors IN THE ORDER given in the docstring:
@@ -1893,8 +1905,7 @@ LONGFORMER_START_DOCSTRING = r"""
Parameters:
config ([`LongformerConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
@@ -1903,9 +1914,8 @@ LONGFORMER_INPUTS_DOCSTRING = r"""
input_ids (`tf.Tensor` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`LongformerTokenizer`]. See
- [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for
- details.
+ Indices can be obtained using [`LongformerTokenizer`]. See [`PreTrainedTokenizer.__call__`] and
+ [`PreTrainedTokenizer.encode`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`tf.Tensor` of shape `({0})`, *optional*):
@@ -1933,20 +1943,22 @@ LONGFORMER_INPUTS_DOCSTRING = r"""
- 1 for global attention (tokens that attend to all other tokens, and all other tokens attend to them).
token_type_ids (`tf.Tensor` of shape `({0})`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`tf.Tensor` of shape `({0})`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
inputs_embeds (`tf.Tensor` of shape `({0}, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the
@@ -1956,8 +1968,8 @@ LONGFORMER_INPUTS_DOCSTRING = r"""
more detail. This argument can be used only in eager mode, in graph mode the value in the config will be
used instead.
return_dict (`bool`, *optional*):
- Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This
- argument can be used in eager mode, in graph mode the value will always be set to True.
+ Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This argument can be used
+ in eager mode, in graph mode the value will always be set to True.
training (`bool`, *optional*, defaults to `False`):
Whether or not to use the model in training mode (some modules like dropout modules have different
behaviors between training and evaluation).
@@ -1971,17 +1983,17 @@ LONGFORMER_INPUTS_DOCSTRING = r"""
class TFLongformerModel(TFLongformerPreTrainedModel):
"""
- This class copies code from [`TFRobertaModel`] and overwrites standard self-attention with
- longformer self-attention to provide the ability to process long sequences following the self-attention approach
- described in [Longformer: the Long-Document Transformer](https://arxiv.org/abs/2004.05150) by Iz Beltagy,
- Matthew E. Peters, and Arman Cohan. Longformer self-attention combines a local (sliding window) and global
- attention to extend to long documents without the O(n^2) increase in memory and compute.
+ This class copies code from [`TFRobertaModel`] and overwrites standard self-attention with longformer
+ self-attention to provide the ability to process long sequences following the self-attention approach described in
+ [Longformer: the Long-Document Transformer](https://arxiv.org/abs/2004.05150) by Iz Beltagy, Matthew E. Peters, and
+ Arman Cohan. Longformer self-attention combines a local (sliding window) and global attention to extend to long
+ documents without the O(n^2) increase in memory and compute.
- The self-attention module `TFLongformerSelfAttention` implemented here supports the combination of local and
- global attention but it lacks support for autoregressive attention and dilated attention. Autoregressive and
- dilated attention are more relevant for autoregressive language modeling than finetuning on downstream tasks.
- Future release will add support for autoregressive attention, but the support for dilated attention requires a
- custom CUDA kernel to be memory and compute efficient.
+ The self-attention module `TFLongformerSelfAttention` implemented here supports the combination of local and global
+ attention but it lacks support for autoregressive attention and dilated attention. Autoregressive and dilated
+ attention are more relevant for autoregressive language modeling than finetuning on downstream tasks. Future
+ release will add support for autoregressive attention, but the support for dilated attention requires a custom CUDA
+ kernel to be memory and compute efficient.
"""
@@ -2053,7 +2065,7 @@ class TFLongformerModel(TFLongformerPreTrainedModel):
@add_start_docstrings(
- """Longformer Model with a `language modeling` head on top. """,
+ """Longformer Model with a `language modeling` head on top.""",
LONGFORMER_START_DOCSTRING,
)
class TFLongformerForMaskedLM(TFLongformerPreTrainedModel, TFMaskedLanguageModelingLoss):
@@ -2099,8 +2111,9 @@ class TFLongformerForMaskedLM(TFLongformerPreTrainedModel, TFMaskedLanguageModel
):
r"""
labels (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
"""
inputs = input_processing(
func=self.call,
@@ -2501,8 +2514,8 @@ class TFLongformerForMultipleChoice(TFLongformerPreTrainedModel, TFMultipleChoic
):
r"""
labels (`tf.Tensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices]` where `num_choices` is the size of the second dimension of the input tensors. (See
- `input_ids` above)
+ Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices]`
+ where `num_choices` is the size of the second dimension of the input tensors. (See `input_ids` above)
"""
inputs = input_processing(
func=self.call,
diff --git a/src/transformers/models/longformer/tokenization_longformer.py b/src/transformers/models/longformer/tokenization_longformer.py
index bba0cbd7c5..19445622b8 100644
--- a/src/transformers/models/longformer/tokenization_longformer.py
+++ b/src/transformers/models/longformer/tokenization_longformer.py
@@ -52,8 +52,8 @@ class LongformerTokenizer(RobertaTokenizer):
r"""
Construct a Longformer tokenizer.
- [`LongformerTokenizer`] is identical to [`RobertaTokenizer`]. Refer to the
- superclass for usage examples and documentation concerning parameters.
+ [`LongformerTokenizer`] is identical to [`RobertaTokenizer`]. Refer to the superclass for usage examples and
+ documentation concerning parameters.
"""
vocab_files_names = VOCAB_FILES_NAMES
pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
diff --git a/src/transformers/models/longformer/tokenization_longformer_fast.py b/src/transformers/models/longformer/tokenization_longformer_fast.py
index 145e90b544..a7d06b1fc3 100644
--- a/src/transformers/models/longformer/tokenization_longformer_fast.py
+++ b/src/transformers/models/longformer/tokenization_longformer_fast.py
@@ -60,8 +60,8 @@ class LongformerTokenizerFast(RobertaTokenizerFast):
r"""
Construct a "fast" Longformer tokenizer (backed by HuggingFace's *tokenizers* library).
- [`LongformerTokenizerFast`] is identical to [`RobertaTokenizerFast`]. Refer
- to the superclass for usage examples and documentation concerning parameters.
+ [`LongformerTokenizerFast`] is identical to [`RobertaTokenizerFast`]. Refer to the superclass for usage examples
+ and documentation concerning parameters.
"""
# merges and vocab same as Roberta
vocab_files_names = VOCAB_FILES_NAMES
diff --git a/src/transformers/models/luke/configuration_luke.py b/src/transformers/models/luke/configuration_luke.py
index 0c39057042..d698f9b922 100644
--- a/src/transformers/models/luke/configuration_luke.py
+++ b/src/transformers/models/luke/configuration_luke.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" LUKE configuration """
+""" LUKE configuration"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging
@@ -28,11 +28,11 @@ LUKE_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class LukeConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`LukeModel`]. It is used to
- instantiate a LUKE model according to the specified arguments, defining the model architecture.
+ This is the configuration class to store the configuration of a [`LukeModel`]. It is used to instantiate a LUKE
+ model according to the specified arguments, defining the model architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
@@ -53,8 +53,8 @@ class LukeConfig(PretrainedConfig):
intermediate_size (`int`, *optional*, defaults to 3072):
Dimensionality of the "intermediate" (often named feed-forward) layer in the Transformer encoder.
hidden_act (`str` or `Callable`, *optional*, defaults to `"gelu"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"silu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"silu"` and `"gelu_new"` are supported.
hidden_dropout_prob (`float`, *optional*, defaults to 0.1):
The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
attention_probs_dropout_prob (`float`, *optional*, defaults to 0.1):
@@ -70,7 +70,8 @@ class LukeConfig(PretrainedConfig):
The epsilon used by the layer normalization layers.
use_entity_aware_attention (`bool`, defaults to `True`):
Whether or not the model should use the entity-aware self-attention mechanism proposed in [LUKE: Deep
- Contextualized Entity Representations with Entity-aware Self-attention (Yamada et al.)](https://arxiv.org/abs/2010.01057).
+ Contextualized Entity Representations with Entity-aware Self-attention (Yamada et
+ al.)](https://arxiv.org/abs/2010.01057).
Examples:
diff --git a/src/transformers/models/luke/modeling_luke.py b/src/transformers/models/luke/modeling_luke.py
index 1a73f73a38..6fe0133be1 100644
--- a/src/transformers/models/luke/modeling_luke.py
+++ b/src/transformers/models/luke/modeling_luke.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-"""PyTorch LUKE model. """
+"""PyTorch LUKE model."""
import math
from dataclasses import dataclass
@@ -62,16 +62,17 @@ class BaseLukeModelOutputWithPooling(BaseModelOutputWithPooling):
Last layer hidden-state of the first token of the sequence (classification token) further processed by a
Linear layer and a Tanh activation function.
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of
- each layer plus the initial embedding outputs.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer
+ plus the initial embedding outputs.
entity_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, entity_length, hidden_size)`. Entity hidden-states of the model at the output
- of each layer plus the initial entity embedding outputs.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, entity_length, hidden_size)`. Entity hidden-states of the model at the output of each
+ layer plus the initial entity embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length + entity_length, sequence_length + entity_length)`. Attentions weights after the attention
- softmax, used to compute the weighted average in the self-attention heads.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length +
+ entity_length, sequence_length + entity_length)`. Attentions weights after the attention softmax, used to
+ compute the weighted average in the self-attention heads.
"""
entity_last_hidden_state: torch.FloatTensor = None
@@ -89,16 +90,17 @@ class BaseLukeModelOutput(BaseModelOutput):
entity_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, entity_length, hidden_size)`):
Sequence of entity hidden-states at the output of the last layer of the model.
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
entity_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, entity_length, hidden_size)`. Entity hidden-states of the model at the output
- of each layer plus the initial entity embedding outputs.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, entity_length, hidden_size)`. Entity hidden-states of the model at the output of each
+ layer plus the initial entity embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -125,16 +127,17 @@ class LukeMaskedLMOutput(ModelOutput):
entity_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
Prediction scores of the entity prediction head (scores for each entity vocabulary token before SoftMax).
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
entity_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, entity_length, hidden_size)`. Entity hidden-states of the model at the output
- of each layer plus the initial entity embedding outputs.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, entity_length, hidden_size)`. Entity hidden-states of the model at the output of each
+ layer plus the initial entity embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -161,16 +164,17 @@ class EntityClassificationOutput(ModelOutput):
logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
Classification scores (before SoftMax).
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of
- each layer plus the initial embedding outputs.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer
+ plus the initial embedding outputs.
entity_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, entity_length, hidden_size)`. Entity hidden-states of the model at the output
- of each layer plus the initial entity embedding outputs.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, entity_length, hidden_size)`. Entity hidden-states of the model at the output of each
+ layer plus the initial entity embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the
- weighted average in the self-attention heads.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
+ the self-attention heads.
"""
loss: Optional[torch.FloatTensor] = None
@@ -191,16 +195,17 @@ class EntityPairClassificationOutput(ModelOutput):
logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
Classification scores (before SoftMax).
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of
- each layer plus the initial embedding outputs.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer
+ plus the initial embedding outputs.
entity_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, entity_length, hidden_size)`. Entity hidden-states of the model at the output
- of each layer plus the initial entity embedding outputs.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, entity_length, hidden_size)`. Entity hidden-states of the model at the output of each
+ layer plus the initial entity embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the
- weighted average in the self-attention heads.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
+ the self-attention heads.
"""
loss: Optional[torch.FloatTensor] = None
@@ -221,16 +226,17 @@ class EntitySpanClassificationOutput(ModelOutput):
logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
Classification scores (before SoftMax).
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of
- each layer plus the initial embedding outputs.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer
+ plus the initial embedding outputs.
entity_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, entity_length, hidden_size)`. Entity hidden-states of the model at the output
- of each layer plus the initial entity embedding outputs.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, entity_length, hidden_size)`. Entity hidden-states of the model at the output of each
+ layer plus the initial entity embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the
- weighted average in the self-attention heads.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
+ the self-attention heads.
"""
loss: Optional[torch.FloatTensor] = None
@@ -777,19 +783,18 @@ class LukePreTrainedModel(PreTrainedModel):
LUKE_START_DOCSTRING = r"""
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
Parameters:
config ([`LukeConfig`]): Model configuration class with all the parameters of the
model. Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
LUKE_INPUTS_DOCSTRING = r"""
@@ -797,9 +802,8 @@ LUKE_INPUTS_DOCSTRING = r"""
input_ids (`torch.LongTensor` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`LukeTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`LukeTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
@@ -810,23 +814,24 @@ LUKE_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
entity_ids (`torch.LongTensor` of shape `(batch_size, entity_length)`):
Indices of entity tokens in the entity vocabulary.
- Indices can be obtained using [`LukeTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`LukeTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
entity_attention_mask (`torch.FloatTensor` of shape `(batch_size, entity_length)`, *optional*):
Mask to avoid performing attention on padding entity token indices. Mask values selected in `[0, 1]`:
@@ -842,12 +847,13 @@ LUKE_INPUTS_DOCSTRING = r"""
- 1 corresponds to a *portion B* entity token.
entity_position_ids (`torch.LongTensor` of shape `(batch_size, entity_length, max_mention_length)`, *optional*):
- Indices of positions of each input entity in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input entity in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:
@@ -1173,11 +1179,13 @@ class LukeForMaskedLM(LukePreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
entity_labels (`torch.LongTensor` of shape `(batch_size, entity_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
Returns:
@@ -1283,11 +1291,11 @@ class LukeForEntityClassification(LukePreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)` or `(batch_size, num_labels)`, *optional*):
- Labels for computing the classification loss. If the shape is `(batch_size,)`, the cross entropy loss
- is used for the single-label classification. In this case, labels should contain the indices that should be
- in `[0, ..., config.num_labels - 1]`. If the shape is `(batch_size, num_labels)`, the binary
- cross entropy loss is used for the multi-label classification. In this case, labels should only contain
- `[0, 1]`, where 0 and 1 indicate false and true, respectively.
+ Labels for computing the classification loss. If the shape is `(batch_size,)`, the cross entropy loss is
+ used for the single-label classification. In this case, labels should contain the indices that should be in
+ `[0, ..., config.num_labels - 1]`. If the shape is `(batch_size, num_labels)`, the binary cross entropy
+ loss is used for the multi-label classification. In this case, labels should only contain `[0, 1]`, where 0
+ and 1 indicate false and true, respectively.
Returns:
@@ -1398,11 +1406,11 @@ class LukeForEntityPairClassification(LukePreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)` or `(batch_size, num_labels)`, *optional*):
- Labels for computing the classification loss. If the shape is `(batch_size,)`, the cross entropy loss
- is used for the single-label classification. In this case, labels should contain the indices that should be
- in `[0, ..., config.num_labels - 1]`. If the shape is `(batch_size, num_labels)`, the binary
- cross entropy loss is used for the multi-label classification. In this case, labels should only contain
- `[0, 1]`, where 0 and 1 indicate false and true, respectively.
+ Labels for computing the classification loss. If the shape is `(batch_size,)`, the cross entropy loss is
+ used for the single-label classification. In this case, labels should contain the indices that should be in
+ `[0, ..., config.num_labels - 1]`. If the shape is `(batch_size, num_labels)`, the binary cross entropy
+ loss is used for the multi-label classification. In this case, labels should only contain `[0, 1]`, where 0
+ and 1 indicate false and true, respectively.
Returns:
@@ -1525,7 +1533,8 @@ class LukeForEntitySpanClassification(LukePreTrainedModel):
labels (`torch.LongTensor` of shape `(batch_size, entity_length)` or `(batch_size, entity_length, num_labels)`, *optional*):
Labels for computing the classification loss. If the shape is `(batch_size, entity_length)`, the cross
entropy loss is used for the single-label classification. In this case, labels should contain the indices
- that should be in `[0, ..., config.num_labels - 1]`. If the shape is `(batch_size, entity_length, num_labels)`, the binary cross entropy loss is used for the multi-label classification. In this case,
+ that should be in `[0, ..., config.num_labels - 1]`. If the shape is `(batch_size, entity_length,
+ num_labels)`, the binary cross entropy loss is used for the multi-label classification. In this case,
labels should only contain `[0, 1]`, where 0 and 1 indicate false and true, respectively.
Returns:
diff --git a/src/transformers/models/luke/tokenization_luke.py b/src/transformers/models/luke/tokenization_luke.py
index 4f86546707..92ff3da4bd 100644
--- a/src/transformers/models/luke/tokenization_luke.py
+++ b/src/transformers/models/luke/tokenization_luke.py
@@ -86,16 +86,15 @@ ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
return_overflowing_tokens (`bool`, *optional*, defaults to `False`):
Whether or not to return overflowing token sequences. If a pair of sequences of input ids (or a batch
- of pairs) is provided with `truncation_strategy = longest_first` or `True`, an error is
- raised instead of returning overflowing tokens.
+ of pairs) is provided with `truncation_strategy = longest_first` or `True`, an error is raised instead
+ of returning overflowing tokens.
return_special_tokens_mask (`bool`, *optional*, defaults to `False`):
Whether or not to return special tokens mask information.
return_offsets_mapping (`bool`, *optional*, defaults to `False`):
Whether or not to return `(char_start, char_end)` for each token.
- This is only available on fast tokenizers inheriting from
- [`PreTrainedTokenizerFast`], if using Python's tokenizer, this method will raise
- `NotImplementedError`.
+ This is only available on fast tokenizers inheriting from [`PreTrainedTokenizerFast`], if using
+ Python's tokenizer, this method will raise `NotImplementedError`.
return_length (`bool`, *optional*, defaults to `False`):
Whether or not to return the lengths of the encoded inputs.
verbose (`bool`, *optional*, defaults to `True`):
@@ -109,8 +108,8 @@ ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING = r"""
[What are input IDs?](../glossary#input-ids)
- - **token_type_ids** -- List of token type ids to be fed to a model (when `return_token_type_ids=True`
- or if *"token_type_ids"* is in `self.model_input_names`).
+ - **token_type_ids** -- List of token type ids to be fed to a model (when `return_token_type_ids=True` or
+ if *"token_type_ids"* is in `self.model_input_names`).
[What are token type IDs?](../glossary#token-type-ids)
@@ -131,8 +130,7 @@ ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING = r"""
[What are token type IDs?](../glossary#token-type-ids)
- **entity_attention_mask** -- List of indices specifying which entities should be attended to by the model
- (when `return_attention_mask=True` or if *"entity_attention_mask"* is in
- `self.model_input_names`).
+ (when `return_attention_mask=True` or if *"entity_attention_mask"* is in `self.model_input_names`).
[What are attention masks?](../glossary#attention-mask)
@@ -155,11 +153,10 @@ class LukeTokenizer(RobertaTokenizer):
r"""
Construct a LUKE tokenizer.
- This tokenizer inherits from [`RobertaTokenizer`] which contains most of the main methods. Users
- should refer to this superclass for more information regarding those methods. Compared to
- [`RobertaTokenizer`], [`LukeTokenizer`] also creates entity sequences, namely
- `entity_ids`, `entity_attention_mask`, `entity_token_type_ids`, and `entity_position_ids` to be
- used by the LUKE model.
+ This tokenizer inherits from [`RobertaTokenizer`] which contains most of the main methods. Users should refer to
+ this superclass for more information regarding those methods. Compared to [`RobertaTokenizer`], [`LukeTokenizer`]
+ also creates entity sequences, namely `entity_ids`, `entity_attention_mask`, `entity_token_type_ids`, and
+ `entity_position_ids` to be used by the LUKE model.
Args:
vocab_file (`str`):
@@ -170,8 +167,8 @@ class LukeTokenizer(RobertaTokenizer):
Path to the entity vocabulary file.
task (`str`, *optional*):
Task for which you want to prepare sequences. One of `"entity_classification"`,
- `"entity_pair_classification"`, or `"entity_span_classification"`. If you specify this argument,
- the entity sequence is automatically created based on the given entity span(s).
+ `"entity_pair_classification"`, or `"entity_span_classification"`. If you specify this argument, the entity
+ sequence is automatically created based on the given entity span(s).
max_entity_length (`int`, *optional*, defaults to 32):
The maximum length of `entity_ids`.
max_mention_length (`int`, *optional*, defaults to 30):
@@ -302,9 +299,9 @@ class LukeTokenizer(RobertaTokenizer):
entity_spans (`List[Tuple[int, int]]`, `List[List[Tuple[int, int]]]`, *optional*):
The sequence or batch of sequences of entity spans to be encoded. Each sequence consists of tuples each
with two integers denoting character-based start and end positions of entities. If you specify
- `"entity_classification"` or `"entity_pair_classification"` as the `task` argument in the
- constructor, the length of each sequence must be 1 or 2, respectively. If you specify `entities`, the
- length of each sequence must be equal to the length of each sequence of `entities`.
+ `"entity_classification"` or `"entity_pair_classification"` as the `task` argument in the constructor,
+ the length of each sequence must be 1 or 2, respectively. If you specify `entities`, the length of each
+ sequence must be equal to the length of each sequence of `entities`.
entity_spans_pair (`List[Tuple[int, int]]`, `List[List[Tuple[int, int]]]`, *optional*):
The sequence or batch of sequences of entity spans to be encoded. Each sequence consists of tuples each
with two integers denoting character-based start and end positions of entities. If you specify the
@@ -313,15 +310,15 @@ class LukeTokenizer(RobertaTokenizer):
entities (`List[str]`, `List[List[str]]`, *optional*):
The sequence or batch of sequences of entities to be encoded. Each sequence consists of strings
representing entities, i.e., special entities (e.g., [MASK]) or entity titles of Wikipedia (e.g., Los
- Angeles). This argument is ignored if you specify the `task` argument in the constructor. The length
- of each sequence must be equal to the length of each sequence of `entity_spans`. If you specify
+ Angeles). This argument is ignored if you specify the `task` argument in the constructor. The length of
+ each sequence must be equal to the length of each sequence of `entity_spans`. If you specify
`entity_spans` without specifying this argument, the entity sequence or the batch of entity sequences
is automatically constructed by filling it with the [MASK] entity.
entities_pair (`List[str]`, `List[List[str]]`, *optional*):
The sequence or batch of sequences of entities to be encoded. Each sequence consists of strings
representing entities, i.e., special entities (e.g., [MASK]) or entity titles of Wikipedia (e.g., Los
- Angeles). This argument is ignored if you specify the `task` argument in the constructor. The length
- of each sequence must be equal to the length of each sequence of `entity_spans_pair`. If you specify
+ Angeles). This argument is ignored if you specify the `task` argument in the constructor. The length of
+ each sequence must be equal to the length of each sequence of `entity_spans_pair`. If you specify
`entity_spans_pair` without specifying this argument, the entity sequence or the batch of entity
sequences is automatically constructed by filling it with the [MASK] entity.
max_entity_length (`int`, *optional*):
@@ -1100,27 +1097,28 @@ class LukeTokenizer(RobertaTokenizer):
"""
Pad a single encoded input or a batch of encoded inputs up to predefined length or to the max sequence length
in the batch. Padding side (left/right) padding token ids are defined at the tokenizer level (with
- `self.padding_side`, `self.pad_token_id` and `self.pad_token_type_id`) .. note:: If the
- `encoded_inputs` passed are dictionary of numpy arrays, PyTorch tensors or TensorFlow tensors, the result
- will use the same type unless you provide a different tensor type with `return_tensors`. In the case of
- PyTorch tensors, you will lose the specific device of your tensors however.
+ `self.padding_side`, `self.pad_token_id` and `self.pad_token_type_id`) .. note:: If the `encoded_inputs` passed
+ are dictionary of numpy arrays, PyTorch tensors or TensorFlow tensors, the result will use the same type unless
+ you provide a different tensor type with `return_tensors`. In the case of PyTorch tensors, you will lose the
+ specific device of your tensors however.
Args:
encoded_inputs ([`BatchEncoding`], list of [`BatchEncoding`], `Dict[str, List[int]]`, `Dict[str, List[List[int]]` or `List[Dict[str, List[int]]]`):
- Tokenized inputs. Can represent one input ([`BatchEncoding`] or `Dict[str, List[int]]`) or a batch of tokenized inputs (list of [`BatchEncoding`], *Dict[str,
- List[List[int]]]* or *List[Dict[str, List[int]]]*) so you can use this method during preprocessing as
- well as in a PyTorch Dataloader collate function. Instead of `List[int]` you can have tensors
- (numpy arrays, PyTorch tensors or TensorFlow tensors), see the note above for the return type.
+ Tokenized inputs. Can represent one input ([`BatchEncoding`] or `Dict[str, List[int]]`) or a batch of
+ tokenized inputs (list of [`BatchEncoding`], *Dict[str, List[List[int]]]* or *List[Dict[str,
+ List[int]]]*) so you can use this method during preprocessing as well as in a PyTorch Dataloader
+ collate function. Instead of `List[int]` you can have tensors (numpy arrays, PyTorch tensors or
+ TensorFlow tensors), see the note above for the return type.
padding (`bool`, `str` or [`~file_utils.PaddingStrategy`], *optional*, defaults to `True`):
Select a strategy to pad the returned sequences (according to the model's padding side and padding
index) among:
- - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a
- single sequence if provided).
- - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the
- maximum acceptable input length for the model if that argument is not provided.
- - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of
- different lengths).
+ - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
+ sequence if provided).
+ - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
+ acceptable input length for the model if that argument is not provided.
+ - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different
+ lengths).
max_length (`int`, *optional*):
Maximum length of the returned list and optionally padding length (see above).
max_entity_length (`int`, *optional*):
@@ -1130,8 +1128,8 @@ class LukeTokenizer(RobertaTokenizer):
the use of Tensor Cores on NVIDIA hardware with compute capability >= 7.5 (Volta).
return_attention_mask (`bool`, *optional*):
Whether to return the attention mask. If left to the default, will return the attention mask according
- to the specific tokenizer's default, defined by the `return_outputs` attribute. [What are
- attention masks?](../glossary#attention-mask)
+ to the specific tokenizer's default, defined by the `return_outputs` attribute. [What are attention
+ masks?](../glossary#attention-mask)
return_tensors (`str` or [`~file_utils.TensorType`], *optional*):
If set, will return tensors instead of list of python integers. Acceptable values are:
@@ -1253,7 +1251,8 @@ class LukeTokenizer(RobertaTokenizer):
Args:
- encoded_inputs: Dictionary of tokenized inputs (`List[int]`) or batch of tokenized inputs (`List[List[int]]`).
+ encoded_inputs:
+ Dictionary of tokenized inputs (`List[int]`) or batch of tokenized inputs (`List[List[int]]`).
max_length: maximum length of the returned list and optionally padding length (see below).
Will truncate by taking into account the special tokens.
max_entity_length: The maximum length of the entity sequence.
@@ -1271,7 +1270,8 @@ class LukeTokenizer(RobertaTokenizer):
pad_to_multiple_of: (optional) Integer if set will pad the sequence to a multiple of the provided value.
This is especially useful to enable the use of Tensor Core on NVIDIA hardware with compute capability
>= 7.5 (Volta).
- return_attention_mask: (optional) Set to False to avoid returning attention mask (default: set to model specifics)
+ return_attention_mask:
+ (optional) Set to False to avoid returning attention mask (default: set to model specifics)
"""
entities_provided = bool("entity_ids" in encoded_inputs)
diff --git a/src/transformers/models/lxmert/configuration_lxmert.py b/src/transformers/models/lxmert/configuration_lxmert.py
index a0fabca535..03b934bb9e 100644
--- a/src/transformers/models/lxmert/configuration_lxmert.py
+++ b/src/transformers/models/lxmert/configuration_lxmert.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" LXMERT model configuration """
+""" LXMERT model configuration"""
from ...configuration_utils import PretrainedConfig
@@ -28,19 +28,17 @@ LXMERT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class LxmertConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`LxmertModel`] or a
- [`TFLxmertModel`]. It is used to instantiate a LXMERT model according to the specified
- arguments, defining the model architecture.
+ This is the configuration class to store the configuration of a [`LxmertModel`] or a [`TFLxmertModel`]. It is used
+ to instantiate a LXMERT model according to the specified arguments, defining the model architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
vocab_size (`int`, *optional*, defaults to 30522):
Vocabulary size of the LXMERT model. Defines the number of different tokens that can be represented by the
- `inputs_ids` passed when calling [`LxmertModel`] or
- [`TFLxmertModel`].
+ `inputs_ids` passed when calling [`LxmertModel`] or [`TFLxmertModel`].
hidden_size (`int`, *optional*, defaults to 768):
Dimensionality of the encoder layers and the pooler layer.
r_layers (`int`, *optional*, defaults to 5):
@@ -54,8 +52,8 @@ class LxmertConfig(PretrainedConfig):
intermediate_size (`int`, *optional*, defaults to 3072):
Dimensionality of the "intermediate" (often named feed-forward) layer in the Transformer encoder.
hidden_act (`str` or `Callable`, *optional*, defaults to `"gelu"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"silu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"silu"` and `"gelu_new"` are supported.
hidden_dropout_prob (`float`, *optional*, defaults to 0.1):
The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
attention_probs_dropout_prob (`float`, *optional*, defaults to 0.1):
diff --git a/src/transformers/models/lxmert/modeling_lxmert.py b/src/transformers/models/lxmert/modeling_lxmert.py
index a4361aeb74..8d8ebb0e41 100644
--- a/src/transformers/models/lxmert/modeling_lxmert.py
+++ b/src/transformers/models/lxmert/modeling_lxmert.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" PyTorch LXMERT model. """
+""" PyTorch LXMERT model."""
import math
@@ -74,20 +74,23 @@ class LxmertModelOutput(ModelOutput):
Last layer hidden-state of the first token of the sequence (classification, CLS, token) further processed
by a Linear layer and a Tanh activation function. The Linear
language_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for input features + one for the output of each cross-modality
- layer) of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for input features + one for the output of each cross-modality layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
vision_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for input features + one for the output of each cross-modality
- layer) of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for input features + one for the output of each cross-modality layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
language_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the
- weighted average in the self-attention heads.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
+ the self-attention heads.
vision_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the
- weighted average in the self-attention heads.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
+ the self-attention heads.
cross_encoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the
- weighted average in the self-attention heads.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
+ the self-attention heads.
"""
language_output: Optional[torch.FloatTensor] = None
@@ -112,20 +115,23 @@ class LxmertForQuestionAnsweringOutput(ModelOutput):
question_answering_score: (`torch.FloatTensor` of shape `(batch_size, n_qa_answers)`, *optional*):
Prediction scores of question answering objective (classification).
language_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for input features + one for the output of each cross-modality
- layer) of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for input features + one for the output of each cross-modality layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
vision_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for input features + one for the output of each cross-modality
- layer) of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for input features + one for the output of each cross-modality layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
language_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the
- weighted average in the self-attention heads.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
+ the self-attention heads.
vision_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the
- weighted average in the self-attention heads.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
+ the self-attention heads.
cross_encoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the
- weighted average in the self-attention heads.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
+ the self-attention heads.
"""
loss: Optional[torch.FloatTensor] = None
@@ -154,20 +160,23 @@ class LxmertForPreTrainingOutput(ModelOutput):
question_answering_score: (`torch.FloatTensor` of shape `(batch_size, n_qa_answers)`):
Prediction scores of question answering objective (classification).
language_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for input features + one for the output of each cross-modality
- layer) of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for input features + one for the output of each cross-modality layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
vision_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for input features + one for the output of each cross-modality
- layer) of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for input features + one for the output of each cross-modality layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
language_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the
- weighted average in the self-attention heads.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
+ the self-attention heads.
vision_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the
- weighted average in the self-attention heads.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
+ the self-attention heads.
cross_encoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the
- weighted average in the self-attention heads.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
+ the self-attention heads.
"""
@@ -792,24 +801,24 @@ class LxmertPreTrainedModel(PreTrainedModel):
LXMERT_START_DOCSTRING = r"""
- The LXMERT model was proposed in [LXMERT: Learning Cross-Modality Encoder Representations from Transformers](https://arxiv.org/abs/1908.07490) by Hao Tan and Mohit Bansal. It's a vision and language transformer model,
- pretrained on a variety of multi-modal datasets comprising of GQA, VQAv2.0, MCSCOCO captions, and Visual genome,
- using a combination of masked language modeling, region of interest feature regression, cross entropy loss for
- question answering attribute prediction, and object tag prediction.
+ The LXMERT model was proposed in [LXMERT: Learning Cross-Modality Encoder Representations from
+ Transformers](https://arxiv.org/abs/1908.07490) by Hao Tan and Mohit Bansal. It's a vision and language transformer
+ model, pretrained on a variety of multi-modal datasets comprising of GQA, VQAv2.0, MCSCOCO captions, and Visual
+ genome, using a combination of masked language modeling, region of interest feature regression, cross entropy loss
+ for question answering attribute prediction, and object tag prediction.
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
Parameters:
config ([`LxmertConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
LXMERT_INPUTS_DOCSTRING = r"""
@@ -818,9 +827,8 @@ LXMERT_INPUTS_DOCSTRING = r"""
input_ids (`torch.LongTensor` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`LxmertTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`LxmertTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
visual_feats: (`torch.FloatTensor` of shape `(batch_size, num_visual_features, visual_feat_dim)`):
@@ -849,16 +857,17 @@ LXMERT_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -1010,7 +1019,7 @@ class LxmertModel(LxmertPreTrainedModel):
@add_start_docstrings(
- """Lxmert Model with a specified pretraining head on top. """,
+ """Lxmert Model with a specified pretraining head on top.""",
LXMERT_START_DOCSTRING,
)
class LxmertForPreTraining(LxmertPreTrainedModel):
@@ -1077,9 +1086,8 @@ class LxmertForPreTraining(LxmertPreTrainedModel):
Args:
num_labels (`int`, *optional*):
New number of labels in the linear layer weight matrix. Increasing the size will add newly initialized
- weights at the end. Reducing the size will remove weights from the end. If not provided or `None`,
- just returns a pointer to the qa labels ``torch.nn.Linear``` module of the model without doing
- anything.
+ weights at the end. Reducing the size will remove weights from the end. If not provided or `None`, just
+ returns a pointer to the qa labels ``torch.nn.Linear``` module of the model without doing anything.
Return:
`torch.nn.Linear`: Pointer to the resized Linear layer or the old Linear layer
@@ -1105,8 +1113,8 @@ class LxmertForPreTraining(LxmertPreTrainedModel):
Returns the the linear layer that produces question answering logits.
Returns:
- `nn.Module`: A torch module mapping the question answering prediction hidden states or `None` if
- LXMERT does not have a visual answering head.
+ `nn.Module`: A torch module mapping the question answering prediction hidden states or `None` if LXMERT
+ does not have a visual answering head.
"""
if hasattr(self, "answer_head"):
return self.answer_head.logit_fc[-1]
@@ -1164,12 +1172,13 @@ class LxmertForPreTraining(LxmertPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
obj_labels: (`Dict[Str: Tuple[Torch.FloatTensor, Torch.FloatTensor]]`, *optional*):
each key is named after each one of the visual losses and each element of the tuple is of the shape
- `(batch_size, num_features)` and `(batch_size, num_features, visual_feature_dim)` for each the label id
- and the label score respectively
+ `(batch_size, num_features)` and `(batch_size, num_features, visual_feature_dim)` for each the label id and
+ the label score respectively
matched_label (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for computing the whether or not the text input matches the image (classification) loss. Input
should be a sequence pair (see `input_ids` docstring) Indices should be in `[0, 1]`:
@@ -1307,9 +1316,8 @@ class LxmertForQuestionAnswering(LxmertPreTrainedModel):
Args:
num_labels (`int`, *optional*):
New number of labels in the linear layer weight matrix. Increasing the size will add newly initialized
- weights at the end. Reducing the size will remove weights from the end. If not provided or `None`,
- just returns a pointer to the qa labels ``torch.nn.Linear``` module of the model without doing
- anything.
+ weights at the end. Reducing the size will remove weights from the end. If not provided or `None`, just
+ returns a pointer to the qa labels ``torch.nn.Linear``` module of the model without doing anything.
Return:
`torch.nn.Linear`: Pointer to the resized Linear layer or the old Linear layer
@@ -1335,8 +1343,8 @@ class LxmertForQuestionAnswering(LxmertPreTrainedModel):
Returns the the linear layer that produces question answering logits
Returns:
- `nn.Module`: A torch module mapping the question answering prediction hidden states. `None`: A
- NoneType object if Lxmert does not have the visual answering head.
+ `nn.Module`: A torch module mapping the question answering prediction hidden states. `None`: A NoneType
+ object if Lxmert does not have the visual answering head.
"""
if hasattr(self, "answer_head"):
diff --git a/src/transformers/models/lxmert/modeling_tf_lxmert.py b/src/transformers/models/lxmert/modeling_tf_lxmert.py
index bb6f79690e..b9f7c12764 100644
--- a/src/transformers/models/lxmert/modeling_tf_lxmert.py
+++ b/src/transformers/models/lxmert/modeling_tf_lxmert.py
@@ -14,7 +14,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" TF 2.0 LXMERT model. """
+""" TF 2.0 LXMERT model."""
import warnings
from dataclasses import dataclass
@@ -63,19 +63,22 @@ class TFLxmertModelOutput(ModelOutput):
Last layer hidden-state of the first token of the sequence (classification, CLS, token) further processed
by a Linear layer and a Tanh activation function. The Linear
language_hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for input features + one for the output of each cross-modality layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for input features + one for the output of each cross-modality layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
vision_hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for input features + one for the output of each cross-modality layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for input features + one for the output of each cross-modality layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
language_attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
the self-attention heads.
vision_attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
the self-attention heads.
cross_encoder_attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
the self-attention heads.
"""
@@ -106,19 +109,22 @@ class TFLxmertForPreTrainingOutput(ModelOutput):
question_answering_score: (`tf.Tensor` of shape `(batch_size, n_qa_answers)`):
Prediction scores of question answering objective (classification).
language_hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for input features + one for the output of each cross-modality layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for input features + one for the output of each cross-modality layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
vision_hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for input features + one for the output of each cross-modality layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for input features + one for the output of each cross-modality layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
language_attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
the self-attention heads.
vision_attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
the self-attention heads.
cross_encoder_attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
the self-attention heads.
"""
@@ -831,14 +837,15 @@ class TFLxmertPreTrainedModel(TFPreTrainedModel):
LXMERT_START_DOCSTRING = r"""
- The LXMERT model was proposed in [LXMERT: Learning Cross-Modality Encoder Representations from Transformers](https://arxiv.org/abs/1908.07490) by Hao Tan and Mohit Bansal. It's a vision and language transformer model,
- pre-trained on a variety of multi-modal datasets comprising of GQA, VQAv2.0, MCSCOCO captions, and Visual genome,
- using a combination of masked language modeling, region of interest feature regression, cross entropy loss for
- question answering attribute prediction, and object tag prediction.
+ The LXMERT model was proposed in [LXMERT: Learning Cross-Modality Encoder Representations from
+ Transformers](https://arxiv.org/abs/1908.07490) by Hao Tan and Mohit Bansal. It's a vision and language transformer
+ model, pre-trained on a variety of multi-modal datasets comprising of GQA, VQAv2.0, MCSCOCO captions, and Visual
+ genome, using a combination of masked language modeling, region of interest feature regression, cross entropy loss
+ for question answering attribute prediction, and object tag prediction.
- This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use
- it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage
- and behavior.
+ This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it
+ as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and
+ behavior.
@@ -847,11 +854,11 @@ LXMERT_START_DOCSTRING = r"""
- having all inputs as keyword arguments (like PyTorch models), or
- having all inputs as a list, tuple or dict in the first positional arguments.
- This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all
- the tensors in the first argument of the model call function: `model(inputs)`.
+ This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all the
+ tensors in the first argument of the model call function: `model(inputs)`.
- If you choose this second option, there are three possibilities you can use to gather all the input Tensors in
- the first positional argument :
+ If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the
+ first positional argument :
- a single Tensor with `input_ids` only and nothing else: `model(inputs_ids)`
- a list of varying length with one or several input Tensors IN THE ORDER given in the docstring:
@@ -864,8 +871,7 @@ LXMERT_START_DOCSTRING = r"""
Parameters:
config ([`LxmertConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
LXMERT_INPUTS_DOCSTRING = r"""
@@ -873,9 +879,8 @@ LXMERT_INPUTS_DOCSTRING = r"""
input_ids (`np.ndarray` or `tf.Tensor` of shape `(batch_size, sequence_length)`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`LxmertTokenizer`]. See
- [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for
- details.
+ Indices can be obtained using [`LxmertTokenizer`]. See [`PreTrainedTokenizer.__call__`] and
+ [`PreTrainedTokenizer.encode`] for details.
[What are input IDs?](../glossary#input-ids)
visual_feats: (`tf.Tensor` of shape `(batch_size, num_visual_features, visual_feat_dim)`):
@@ -904,16 +909,17 @@ LXMERT_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
inputs_embeds (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the
@@ -923,8 +929,8 @@ LXMERT_INPUTS_DOCSTRING = r"""
more detail. This argument can be used only in eager mode, in graph mode the value in the config will be
used instead.
return_dict (`bool`, *optional*):
- Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This
- argument can be used in eager mode, in graph mode the value will always be set to True.
+ Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This argument can be used
+ in eager mode, in graph mode the value will always be set to True.
training (`bool`, *optional*, defaults to `False`):
Whether or not to use the model in training mode (some modules like dropout modules have different
behaviors between training and evaluation).
@@ -1191,7 +1197,7 @@ class TFLxmertVisualObjHead(tf.keras.layers.Layer):
return output
-@add_start_docstrings("""Lxmert Model with a `language modeling` head on top. """, LXMERT_START_DOCSTRING)
+@add_start_docstrings("""Lxmert Model with a `language modeling` head on top.""", LXMERT_START_DOCSTRING)
class TFLxmertForPreTraining(TFLxmertPreTrainedModel):
def __init__(self, config, *inputs, **kwargs):
super().__init__(config, *inputs, **kwargs)
@@ -1315,12 +1321,13 @@ class TFLxmertForPreTraining(TFLxmertPreTrainedModel):
):
r"""
masked_lm_labels (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
obj_labels: (`Dict[Str: Tuple[tf.Tensor, tf.Tensor]]`, *optional*, defaults to :obj: *None*):
each key is named after each one of the visual losses and each element of the tuple is of the shape
- `(batch_size, num_features)` and `(batch_size, num_features, visual_feature_dim)` for each the label id
- and the label score respectively
+ `(batch_size, num_features)` and `(batch_size, num_features, visual_feature_dim)` for each the label id and
+ the label score respectively
matched_label (`tf.Tensor` of shape `(batch_size,)`, *optional*):
Labels for computing the whether or not the text input matches the image (classification) loss. Input
should be a sequence pair (see `input_ids` docstring) Indices should be in `[0, 1]`:
diff --git a/src/transformers/models/lxmert/tokenization_lxmert.py b/src/transformers/models/lxmert/tokenization_lxmert.py
index 5d4e97ad54..a5e5d6a6c9 100644
--- a/src/transformers/models/lxmert/tokenization_lxmert.py
+++ b/src/transformers/models/lxmert/tokenization_lxmert.py
@@ -37,11 +37,10 @@ class LxmertTokenizer(BertTokenizer):
r"""
Construct an LXMERT tokenizer.
- [`LxmertTokenizer`] is identical to [`BertTokenizer`] and runs end-to-end
- tokenization: punctuation splitting and wordpiece.
+ [`LxmertTokenizer`] is identical to [`BertTokenizer`] and runs end-to-end tokenization: punctuation splitting and
+ wordpiece.
- Refer to superclass [`BertTokenizer`] for usage examples and documentation concerning
- parameters.
+ Refer to superclass [`BertTokenizer`] for usage examples and documentation concerning parameters.
"""
vocab_files_names = VOCAB_FILES_NAMES
diff --git a/src/transformers/models/lxmert/tokenization_lxmert_fast.py b/src/transformers/models/lxmert/tokenization_lxmert_fast.py
index 08323d25f6..9e88bc1581 100644
--- a/src/transformers/models/lxmert/tokenization_lxmert_fast.py
+++ b/src/transformers/models/lxmert/tokenization_lxmert_fast.py
@@ -41,11 +41,10 @@ class LxmertTokenizerFast(BertTokenizerFast):
r"""
Construct a "fast" LXMERT tokenizer (backed by HuggingFace's *tokenizers* library).
- [`LxmertTokenizerFast`] is identical to [`BertTokenizerFast`] and runs
- end-to-end tokenization: punctuation splitting and wordpiece.
+ [`LxmertTokenizerFast`] is identical to [`BertTokenizerFast`] and runs end-to-end tokenization: punctuation
+ splitting and wordpiece.
- Refer to superclass [`BertTokenizerFast`] for usage examples and documentation concerning
- parameters.
+ Refer to superclass [`BertTokenizerFast`] for usage examples and documentation concerning parameters.
"""
vocab_files_names = VOCAB_FILES_NAMES
pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
diff --git a/src/transformers/models/m2m_100/configuration_m2m_100.py b/src/transformers/models/m2m_100/configuration_m2m_100.py
index 3651f51487..a383be9f25 100644
--- a/src/transformers/models/m2m_100/configuration_m2m_100.py
+++ b/src/transformers/models/m2m_100/configuration_m2m_100.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" M2M100 model configuration """
+""" M2M100 model configuration"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging
@@ -28,12 +28,13 @@ M2M_100_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class M2M100Config(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`M2M100Model`]. It is used to
- instantiate an M2M100 model according to the specified arguments, defining the model architecture. Instantiating a
- configuration with the defaults will yield a similar configuration to that of the M2M100 [m2m100_418M](https://huggingface.co/facebook/m2m100_418M) architecture.
+ This is the configuration class to store the configuration of a [`M2M100Model`]. It is used to instantiate an
+ M2M100 model according to the specified arguments, defining the model architecture. Instantiating a configuration
+ with the defaults will yield a similar configuration to that of the M2M100
+ [m2m100_418M](https://huggingface.co/facebook/m2m100_418M) architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
@@ -55,8 +56,8 @@ class M2M100Config(PretrainedConfig):
encoder_ffn_dim (`int`, *optional*, defaults to 4096):
Dimensionality of the "intermediate" (often named feed-forward) layer in decoder.
activation_function (`str` or `function`, *optional*, defaults to `"gelu"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"silu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"silu"` and `"gelu_new"` are supported.
dropout (`float`, *optional*, defaults to 0.1):
The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
attention_dropout (`float`, *optional*, defaults to 0.0):
@@ -71,11 +72,11 @@ class M2M100Config(PretrainedConfig):
init_std (`float`, *optional*, defaults to 0.02):
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
encoder_layerdrop: (`float`, *optional*, defaults to 0.0):
- The LayerDrop probability for the encoder. See the [LayerDrop paper](see
- https://arxiv.org/abs/1909.11556) for more details.
+ The LayerDrop probability for the encoder. See the [LayerDrop paper](see https://arxiv.org/abs/1909.11556)
+ for more details.
decoder_layerdrop: (`float`, *optional*, defaults to 0.0):
- The LayerDrop probability for the decoder. See the [LayerDrop paper](see
- https://arxiv.org/abs/1909.11556) for more details.
+ The LayerDrop probability for the decoder. See the [LayerDrop paper](see https://arxiv.org/abs/1909.11556)
+ for more details.
use_cache (`bool`, *optional*, defaults to `True`):
Whether or not the model should return the last key/values attentions (not used by all models).
diff --git a/src/transformers/models/m2m_100/modeling_m2m_100.py b/src/transformers/models/m2m_100/modeling_m2m_100.py
index 21d86aa2e0..c5900c0020 100755
--- a/src/transformers/models/m2m_100/modeling_m2m_100.py
+++ b/src/transformers/models/m2m_100/modeling_m2m_100.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" PyTorch M2M100 model. """
+""" PyTorch M2M100 model."""
import math
@@ -453,7 +453,8 @@ class M2M100DecoderLayer(nn.Module):
hidden_states (`torch.FloatTensor`): input to the layer of shape *(seq_len, batch, embed_dim)*
attention_mask (`torch.FloatTensor`): attention mask of size
*(batch, 1, tgt_len, src_len)* where padding elements are indicated by very large negative values.
- encoder_hidden_states (`torch.FloatTensor`): cross attention input to the layer of shape *(seq_len, batch, embed_dim)*
+ encoder_hidden_states (`torch.FloatTensor`):
+ cross attention input to the layer of shape *(seq_len, batch, embed_dim)*
encoder_attention_mask (`torch.FloatTensor`): encoder attention mask of size
*(batch, 1, tgt_len, src_len)* where padding elements are indicated by very large negative values.
layer_head_mask (`torch.FloatTensor`): mask for attention heads in a given layer of size
@@ -547,13 +548,13 @@ class M2M100PreTrainedModel(PreTrainedModel):
M2M_100_START_DOCSTRING = r"""
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
Parameters:
config ([`M2M100Config`]):
@@ -567,15 +568,15 @@ M2M_100_GENERATION_EXAMPLE = r"""
>>> from transformers import M2M100Tokenizer, M2M100ForConditionalGeneration
- >>> model = M2M100ForConditionalGeneration.from_pretrained('facebook/m2m100_418M')
- >>> tokenizer = M2M100Tokenizer.from_pretrained('facebook/m2m100_418M')
+ >>> model = M2M100ForConditionalGeneration.from_pretrained('facebook/m2m100_418M') >>> tokenizer =
+ M2M100Tokenizer.from_pretrained('facebook/m2m100_418M')
- >>> text_to_translate = "Life is like a box of chocolates"
- >>> model_inputs = tokenizer(text_to_translate, return_tensors='pt')
+ >>> text_to_translate = "Life is like a box of chocolates" >>> model_inputs = tokenizer(text_to_translate,
+ return_tensors='pt')
- >>> # translate to French
- >>> gen_tokens = model.generate( **model_inputs, forced_bos_token_id=tokenizer.get_lang_id("fr"))
- >>> print(tokenizer.batch_decode(gen_tokens, skip_special_tokens=True))
+ >>> # translate to French >>> gen_tokens = model.generate( **model_inputs,
+ forced_bos_token_id=tokenizer.get_lang_id("fr")) >>> print(tokenizer.batch_decode(gen_tokens,
+ skip_special_tokens=True))
"""
M2M_100_INPUTS_DOCSTRING = r"""
@@ -584,9 +585,8 @@ M2M_100_INPUTS_DOCSTRING = r"""
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
it.
- Indices can be obtained using [`M2M100Tokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`M2M100Tokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -599,9 +599,8 @@ M2M_100_INPUTS_DOCSTRING = r"""
decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
Indices of decoder input sequence tokens in the vocabulary.
- Indices can be obtained using [`M2M100Tokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`M2M100Tokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are decoder input IDs?](../glossary#decoder-input-ids)
@@ -609,8 +608,8 @@ M2M_100_INPUTS_DOCSTRING = r"""
`past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
`past_key_values`).
decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
- Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will
- also be used by default.
+ Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
+ be used by default.
head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*):
Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`:
@@ -624,38 +623,41 @@ M2M_100_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*):
- Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, 1]`:
+ Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0,
+ 1]`:
- 1 indicates the head is **not masked**,
- 0 indicates the head is **masked**.
encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*):
- Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*:
- `attentions`) `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`,
- *optional*) is a sequence of hidden-states at the output of the last layer of the encoder. Used in the
- cross-attention of the decoder.
+ Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`)
+ `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of
+ hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder.
past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors
- of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
- shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
+ Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
+ `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape
+ `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all ``decoder_input_ids``` of shape `(batch_size, sequence_length)`. inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ ``decoder_input_ids``` of shape `(batch_size, sequence_length)`. inputs_embeds (`torch.FloatTensor` of
+ shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids`
+ you can choose to directly pass an embedded representation. This is useful if you want more control over
+ how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup
+ matrix.
decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*):
Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded
- representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds`
- have to be input (see `past_key_values`). This is useful if you want more control over how to convert
+ representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be
+ input (see `past_key_values`). This is useful if you want more control over how to convert
`decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix.
- If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds`
- takes the value of `inputs_embeds`.
+ If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds` takes the value
+ of `inputs_embeds`.
use_cache (`bool`, *optional*):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`).
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`).
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -721,9 +723,8 @@ class M2M100Encoder(M2M100PreTrainedModel):
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
provide it.
- Indices can be obtained using [`M2M100Tokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`]
- for details.
+ Indices can be obtained using [`M2M100Tokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -740,9 +741,9 @@ class M2M100Encoder(M2M100PreTrainedModel):
- 0 indicates the head is **masked**.
inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded
- representation. This is useful if you want more control over how to convert `input_ids` indices
- into associated vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
+ This is useful if you want more control over how to convert `input_ids` indices into associated vectors
+ than the model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under
returned tensors for more detail.
@@ -892,9 +893,8 @@ class M2M100Decoder(M2M100PreTrainedModel):
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
provide it.
- Indices can be obtained using [`M2M100Tokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`]
- for details.
+ Indices can be obtained using [`M2M100Tokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -929,19 +929,20 @@ class M2M100Decoder(M2M100PreTrainedModel):
- 0 indicates the head is **masked**.
past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2
- tensors of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional
- tensors of shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
+ Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of
+ shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
+ shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
Contains pre-computed hidden-states (key and values in the self-attention blocks and in the
- cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential
- decoding.
+ cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
- If `past_key_values` are used, the user can optionally input only the last
- `decoder_input_ids` (those that don't have their past key value states given to this model) of
- shape `(batch_size, 1)` instead of all ``decoder_input_ids``` of shape `(batch_size,
- sequence_length)`. inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices
- into associated vectors than the model's internal embedding lookup matrix.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those
+ that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of
+ all ``decoder_input_ids``` of shape `(batch_size, sequence_length)`. inputs_embeds (`torch.FloatTensor`
+ of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing
+ `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more
+ control over how to convert `input_ids` indices into associated vectors than the model's internal
+ embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under
returned tensors for more detail.
@@ -1271,7 +1272,8 @@ class M2M100ForConditionalGeneration(M2M100PreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
+ Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
+ config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
(masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
Returns:
diff --git a/src/transformers/models/m2m_100/tokenization_m2m_100.py b/src/transformers/models/m2m_100/tokenization_m2m_100.py
index 7cd7bc4003..80ca021252 100644
--- a/src/transformers/models/m2m_100/tokenization_m2m_100.py
+++ b/src/transformers/models/m2m_100/tokenization_m2m_100.py
@@ -65,15 +65,15 @@ class M2M100Tokenizer(PreTrainedTokenizer):
"""
Construct an M2M100 tokenizer. Based on [SentencePiece](https://github.com/google/sentencepiece).
- This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods.
- Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
+ this superclass for more information regarding those methods.
Args:
vocab_file (`str`):
Path to the vocabulary file.
spm_file (`str`):
- Path to [SentencePiece](https://github.com/google/sentencepiece) file (generally has a .spm extension)
- that contains the vocabulary.
+ Path to [SentencePiece](https://github.com/google/sentencepiece) file (generally has a .spm extension) that
+ contains the vocabulary.
src_lang (`str`, *optional*):
A string representing the source language.
tgt_lang (`str`, *optional*):
@@ -92,7 +92,9 @@ class M2M100Tokenizer(PreTrainedTokenizer):
language_codes (`str`, *optional*, defaults to `"m2m100"`):
What language codes to use. Should be one of `"m2m100"` or `"wmt21"`.
sp_model_kwargs (`dict`, *optional*):
- Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things, to set:
+ Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for
+ SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things,
+ to set:
- `enable_sampling`: Enable subword regularization.
- `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout.
diff --git a/src/transformers/models/marian/configuration_marian.py b/src/transformers/models/marian/configuration_marian.py
index 6ee95889b6..a37e2f2074 100644
--- a/src/transformers/models/marian/configuration_marian.py
+++ b/src/transformers/models/marian/configuration_marian.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" Marian model configuration """
+""" Marian model configuration"""
from collections import OrderedDict
from typing import Any, Mapping, Optional
@@ -34,20 +34,19 @@ MARIAN_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class MarianConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`MarianModel`]. It is used to
- instantiate an Marian model according to the specified arguments, defining the model architecture. Instantiating a
- configuration with the defaults will yield a similar configuration to that of the Marian
+ This is the configuration class to store the configuration of a [`MarianModel`]. It is used to instantiate an
+ Marian model according to the specified arguments, defining the model architecture. Instantiating a configuration
+ with the defaults will yield a similar configuration to that of the Marian
[Helsinki-NLP/opus-mt-en-de](https://huggingface.co/Helsinki-NLP/opus-mt-en-de) architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
vocab_size (`int`, *optional*, defaults to 50265):
Vocabulary size of the Marian model. Defines the number of different tokens that can be represented by the
- `inputs_ids` passed when calling [`MarianModel`] or
- [`TFMarianModel`].
+ `inputs_ids` passed when calling [`MarianModel`] or [`TFMarianModel`].
d_model (`int`, *optional*, defaults to 1024):
Dimensionality of the layers and the pooler layer.
encoder_layers (`int`, *optional*, defaults to 12):
@@ -63,8 +62,8 @@ class MarianConfig(PretrainedConfig):
encoder_ffn_dim (`int`, *optional*, defaults to 4096):
Dimensionality of the "intermediate" (often named feed-forward) layer in decoder.
activation_function (`str` or `function`, *optional*, defaults to `"gelu"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"silu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"silu"` and `"gelu_new"` are supported.
dropout (`float`, *optional*, defaults to 0.1):
The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
attention_dropout (`float`, *optional*, defaults to 0.0):
@@ -79,11 +78,11 @@ class MarianConfig(PretrainedConfig):
init_std (`float`, *optional*, defaults to 0.02):
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
encoder_layerdrop: (`float`, *optional*, defaults to 0.0):
- The LayerDrop probability for the encoder. See the [LayerDrop paper](see
- https://arxiv.org/abs/1909.11556) for more details.
+ The LayerDrop probability for the encoder. See the [LayerDrop paper](see https://arxiv.org/abs/1909.11556)
+ for more details.
decoder_layerdrop: (`float`, *optional*, defaults to 0.0):
- The LayerDrop probability for the decoder. See the [LayerDrop paper](see
- https://arxiv.org/abs/1909.11556) for more details.
+ The LayerDrop probability for the decoder. See the [LayerDrop paper](see https://arxiv.org/abs/1909.11556)
+ for more details.
scale_embedding (`bool`, *optional*, defaults to `False`):
Scale embeddings by diving by sqrt(d_model).
use_cache (`bool`, *optional*, defaults to `True`):
diff --git a/src/transformers/models/marian/modeling_flax_marian.py b/src/transformers/models/marian/modeling_flax_marian.py
index 027eb6633b..0b0c32e591 100644
--- a/src/transformers/models/marian/modeling_flax_marian.py
+++ b/src/transformers/models/marian/modeling_flax_marian.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" Flax Marian model. """
+""" Flax Marian model."""
import math
import random
@@ -57,12 +57,13 @@ _TOKENIZER_FOR_DOC = "MarianTokenizer"
MARIAN_START_DOCSTRING = r"""
- This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading or saving, resizing the input
- embeddings, pruning heads etc.)
+ This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a Flax Linen [flax.nn.Module](https://flax.readthedocs.io/en/latest/_autosummary/flax.nn.module.html) subclass. Use it as a regular Flax
- Module and refer to the Flax documentation for all matter related to general usage and behavior.
+ This model is also a Flax Linen
+ [flax.nn.Module](https://flax.readthedocs.io/en/latest/_autosummary/flax.nn.module.html) subclass. Use it as a
+ regular Flax Module and refer to the Flax documentation for all matter related to general usage and behavior.
Finally, this model supports inherent JAX features such as:
@@ -74,11 +75,10 @@ MARIAN_START_DOCSTRING = r"""
Parameters:
config ([`MarianConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the
- model weights.
+ configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the model weights.
dtype (`jax.numpy.dtype`, *optional*, defaults to `jax.numpy.float32`):
- The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on
- GPUs) and `jax.numpy.bfloat16` (on TPUs).
+ The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on GPUs) and
+ `jax.numpy.bfloat16` (on TPUs).
This can be used to enable mixed-precision training or half-precision inference on GPUs or TPUs. If
specified all the computation will be performed with the given `dtype`.
@@ -86,8 +86,8 @@ MARIAN_START_DOCSTRING = r"""
**Note that this only specifies the dtype of the computation and does not influence the dtype of model
parameters.**
- If you wish to change the dtype of the model parameters, see
- [`~FlaxPreTrainedModel.to_fp16`] and [`~FlaxPreTrainedModel.to_bf16`].
+ If you wish to change the dtype of the model parameters, see [`~FlaxPreTrainedModel.to_fp16`] and
+ [`~FlaxPreTrainedModel.to_bf16`].
"""
MARIAN_INPUTS_DOCSTRING = r"""
@@ -96,9 +96,8 @@ MARIAN_INPUTS_DOCSTRING = r"""
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
it.
- Indices can be obtained using [`MarianTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`MarianTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`jnp.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
@@ -111,22 +110,23 @@ MARIAN_INPUTS_DOCSTRING = r"""
decoder_input_ids (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`, *optional*):
Indices of decoder input sequence tokens in the vocabulary.
- Indices can be obtained using [`MarianTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`MarianTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are decoder input IDs?](../glossary#decoder-input-ids)
For translation and summarization training, `decoder_input_ids` should be provided. If no
- `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to
- the right for denoising pre-training following the paper.
+ `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right
+ for denoising pre-training following the paper.
decoder_attention_mask (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`, *optional*):
- Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will
- also be used by default.
+ Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
+ be used by default.
- If you want to change padding behavior, you should modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy.
+ If you want to change padding behavior, you should modify to your needs. See diagram 1 in [the
+ paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy.
position_ids (`numpy.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
decoder_position_ids (`numpy.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the
range `[0, config.max_position_embeddings - 1]`.
@@ -147,9 +147,8 @@ MARIAN_ENCODE_INPUTS_DOCSTRING = r"""
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
it.
- Indices can be obtained using [`MarianTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`MarianTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`jnp.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
@@ -160,7 +159,8 @@ MARIAN_ENCODE_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
position_ids (`numpy.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -176,20 +176,18 @@ MARIAN_DECODE_INPUTS_DOCSTRING = r"""
decoder_input_ids (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`):
Indices of decoder input sequence tokens in the vocabulary.
- Indices can be obtained using [`MarianTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`MarianTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are decoder input IDs?](../glossary#decoder-input-ids)
For translation and summarization training, `decoder_input_ids` should be provided. If no
- `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to
- the right for denoising pre-training following the paper.
+ `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right
+ for denoising pre-training following the paper.
encoder_outputs (`tuple(tuple(jnp.ndarray)`):
- Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*:
- `attentions`) `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`,
- *optional*) is a sequence of hidden-states at the output of the last layer of the encoder. Used in the
- cross-attention of the decoder.
+ Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`)
+ `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of
+ hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder.
encoder_attention_mask (`jnp.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
@@ -198,10 +196,11 @@ MARIAN_DECODE_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
decoder_attention_mask (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`, *optional*):
- Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will
- also be used by default.
+ Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
+ be used by default.
- If you want to change padding behavior, you should modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy.
+ If you want to change padding behavior, you should modify to your needs. See diagram 1 in [the
+ paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy.
decoder_position_ids (`numpy.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the
range `[0, config.max_position_embeddings - 1]`.
@@ -938,9 +937,10 @@ class FlaxMarianPreTrainedModel(FlaxPreTrainedModel):
maximum possible length for auto-regressive decoding. Defines the sequence length of the initialized
cache.
encoder_outputs (`Union[FlaxBaseModelOutput, tuple(tuple(jnp.ndarray)]`):
- `encoder_outputs` consists of (`last_hidden_state`, *optional*: `hidden_states`,
- *optional*: `attentions`). `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of hidden-states at the output of the last layer of the
- encoder. Used in the cross-attention of the decoder.
+ `encoder_outputs` consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*:
+ `attentions`). `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*)
+ is a sequence of hidden-states at the output of the last layer of the encoder. Used in the
+ cross-attention of the decoder.
"""
# init input variables to retrieve cache
decoder_input_ids = jnp.ones((batch_size, max_length), dtype="i4")
diff --git a/src/transformers/models/marian/modeling_marian.py b/src/transformers/models/marian/modeling_marian.py
index ce3f4f8482..c6be85348a 100755
--- a/src/transformers/models/marian/modeling_marian.py
+++ b/src/transformers/models/marian/modeling_marian.py
@@ -399,7 +399,8 @@ class MarianDecoderLayer(nn.Module):
hidden_states (`torch.FloatTensor`): input to the layer of shape *(batch, seq_len, embed_dim)*
attention_mask (`torch.FloatTensor`): attention mask of size
*(batch, 1, tgt_len, src_len)* where padding elements are indicated by very large negative values.
- encoder_hidden_states (`torch.FloatTensor`): cross attention input to the layer of shape *(batch, seq_len, embed_dim)*
+ encoder_hidden_states (`torch.FloatTensor`):
+ cross attention input to the layer of shape *(batch, seq_len, embed_dim)*
encoder_attention_mask (`torch.FloatTensor`): encoder attention mask of size
*(batch, 1, tgt_len, src_len)* where padding elements are indicated by very large negative values.
layer_head_mask (`torch.FloatTensor`): mask for attention heads in a given layer of size
@@ -506,13 +507,13 @@ class MarianPreTrainedModel(PreTrainedModel):
MARIAN_START_DOCSTRING = r"""
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
Parameters:
config ([`MarianConfig`]):
@@ -550,9 +551,8 @@ MARIAN_INPUTS_DOCSTRING = r"""
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
it.
- Indices can be obtained using [`MarianTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`MarianTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -565,9 +565,8 @@ MARIAN_INPUTS_DOCSTRING = r"""
decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
Indices of decoder input sequence tokens in the vocabulary.
- Indices can be obtained using [`MarianTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`MarianTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are decoder input IDs?](../glossary#decoder-input-ids)
@@ -575,8 +574,8 @@ MARIAN_INPUTS_DOCSTRING = r"""
`past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
`past_key_values`).
decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
- Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will
- also be used by default.
+ Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
+ be used by default.
head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*):
Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`:
@@ -590,42 +589,42 @@ MARIAN_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*):
- Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, 1]`:
+ Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0,
+ 1]`:
- 1 indicates the head is **not masked**,
- 0 indicates the head is **masked**.
encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*):
- Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*:
- `attentions`) `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`,
- *optional*) is a sequence of hidden-states at the output of the last layer of the encoder. Used in the
- cross-attention of the decoder.
+ Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`)
+ `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of
+ hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder.
past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors
- of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
- shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
+ Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
+ `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape
+ `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ `decoder_input_ids` of shape `(batch_size, sequence_length)`.
inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
model's internal embedding lookup matrix.
decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*):
Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded
- representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds`
- have to be input (see `past_key_values`). This is useful if you want more control over how to convert
+ representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be
+ input (see `past_key_values`). This is useful if you want more control over how to convert
`decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix.
- If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds`
- takes the value of `inputs_embeds`.
+ If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds` takes the value
+ of `inputs_embeds`.
use_cache (`bool`, *optional*):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`).
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`).
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -690,9 +689,8 @@ class MarianEncoder(MarianPreTrainedModel):
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
provide it.
- Indices can be obtained using [`MarianTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`]
- for details.
+ Indices can be obtained using [`MarianTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -709,9 +707,9 @@ class MarianEncoder(MarianPreTrainedModel):
- 0 indicates the head is **masked**.
inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded
- representation. This is useful if you want more control over how to convert `input_ids` indices
- into associated vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
+ This is useful if you want more control over how to convert `input_ids` indices into associated vectors
+ than the model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under
returned tensors for more detail.
@@ -883,9 +881,8 @@ class MarianDecoder(MarianPreTrainedModel):
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
provide it.
- Indices can be obtained using [`MarianTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`]
- for details.
+ Indices can be obtained using [`MarianTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -920,18 +917,16 @@ class MarianDecoder(MarianPreTrainedModel):
- 0 indicates the head is **masked**.
past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2
- tensors of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional
- tensors of shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
+ Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of
+ shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
+ shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
Contains pre-computed hidden-states (key and values in the self-attention blocks and in the
- cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential
- decoding.
+ cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
- If `past_key_values` are used, the user can optionally input only the last
- `decoder_input_ids` (those that don't have their past key value states given to this model) of
- shape `(batch_size, 1)` instead of all ``decoder_input_ids``` of shape `(batch_size,
- sequence_length)`.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those
+ that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of
+ all ``decoder_input_ids``` of shape `(batch_size, sequence_length)`.
inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
This is useful if you want more control over how to convert `input_ids` indices into associated vectors
@@ -1280,7 +1275,8 @@ class MarianMTModel(MarianPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
+ Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
+ config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
(masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
Returns:
@@ -1450,9 +1446,8 @@ class MarianForCausalLM(MarianPreTrainedModel):
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
provide it.
- Indices can be obtained using [`MarianTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`]
- for details.
+ Indices can be obtained using [`MarianTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -1481,25 +1476,24 @@ class MarianForCausalLM(MarianPreTrainedModel):
- 0 indicates the head is **masked**.
past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2
- tensors of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional
- tensors of shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. The two
- additional tensors are only required when the model is used as a decoder in a Sequence to Sequence
- model.
+ Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of
+ shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
+ shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. The two additional
+ tensors are only required when the model is used as a decoder in a Sequence to Sequence model.
Contains pre-computed hidden-states (key and values in the self-attention blocks and in the
- cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential
- decoding.
+ cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those
+ that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of
+ all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are
- ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
+ Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
+ config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
+ (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
use_cache (`bool`, *optional*):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`).
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
+ (see `past_key_values`).
- 1 for tokens that are **not masked**,
- 0 for tokens that are **masked**.
diff --git a/src/transformers/models/marian/modeling_tf_marian.py b/src/transformers/models/marian/modeling_tf_marian.py
index f8638c14a3..52c8ac7275 100644
--- a/src/transformers/models/marian/modeling_tf_marian.py
+++ b/src/transformers/models/marian/modeling_tf_marian.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" TF 2.0 Marian model. """
+""" TF 2.0 Marian model."""
import random
@@ -419,7 +419,8 @@ class TFMarianDecoderLayer(tf.keras.layers.Layer):
hidden_states (`tf.Tensor`): input to the layer of shape *(seq_len, batch, embed_dim)*
attention_mask (`tf.Tensor`): attention mask of size
*(batch, 1, tgt_len, src_len)* where padding elements are indicated by very large negative values.
- encoder_hidden_states (`tf.Tensor`): cross attention input to the layer of shape *(seq_len, batch, embed_dim)*
+ encoder_hidden_states (`tf.Tensor`):
+ cross attention input to the layer of shape *(seq_len, batch, embed_dim)*
encoder_attention_mask (`tf.Tensor`): encoder attention mask of size
*(batch, 1, tgt_len, src_len)* where padding elements are indicated by very large negative values.
layer_head_mask (`tf.Tensor`): mask for attention heads in a given layer of size
@@ -517,13 +518,13 @@ class TFMarianPreTrainedModel(TFPreTrainedModel):
MARIAN_START_DOCSTRING = r"""
- This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading or saving, resizing the input
- embeddings, pruning heads etc.)
+ This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use
- it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage
- and behavior.
+ This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it
+ as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and
+ behavior.
@@ -532,11 +533,11 @@ MARIAN_START_DOCSTRING = r"""
- having all inputs as keyword arguments (like PyTorch models), or
- having all inputs as a list, tuple or dict in the first positional arguments.
- This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all
- the tensors in the first argument of the model call function: `model(inputs)`.
+ This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all the
+ tensors in the first argument of the model call function: `model(inputs)`.
- If you choose this second option, there are three possibilities you can use to gather all the input Tensors in
- the first positional argument :
+ If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the
+ first positional argument :
- a single Tensor with `input_ids` only and nothing else: `model(input_ids)`
- a list of varying length with one or several input Tensors IN THE ORDER given in the docstring:
@@ -549,8 +550,7 @@ MARIAN_START_DOCSTRING = r"""
Args:
config ([`MarianConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~TFPreTrainedModel.from_pretrained`] method to load the
- model weights.
+ configuration. Check out the [`~TFPreTrainedModel.from_pretrained`] method to load the model weights.
"""
MARIAN_GENERATION_EXAMPLE = r"""
@@ -581,9 +581,8 @@ MARIAN_INPUTS_DOCSTRING = r"""
input_ids (`tf.Tensor` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`MarianTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`MarianTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`tf.Tensor` of shape `({0})`, *optional*):
@@ -596,9 +595,8 @@ MARIAN_INPUTS_DOCSTRING = r"""
decoder_input_ids (`tf.Tensor` of shape `(batch_size, target_sequence_length)`, *optional*):
Indices of decoder input sequence tokens in the vocabulary.
- Indices can be obtained using [`MarianTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`MarianTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are decoder input IDs?](../glossary#decoder-input-ids)
@@ -630,12 +628,12 @@ MARIAN_INPUTS_DOCSTRING = r"""
of shape `(batch_size, sequence_length, hidden_size)` is a sequence of
past_key_values (`Tuple[Tuple[tf.Tensor]]` of length `config.n_layers`)
contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ `decoder_input_ids` of shape `(batch_size, sequence_length)`.
use_cache (`bool`, *optional*, defaults to `True`):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`). Set to `False` during training, `True` during generation
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`). Set to `False` during training, `True` during generation
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the
@@ -645,8 +643,8 @@ MARIAN_INPUTS_DOCSTRING = r"""
more detail. This argument can be used only in eager mode, in graph mode the value in the config will be
used instead.
return_dict (`bool`, *optional*):
- Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This
- argument can be used in eager mode, in graph mode the value will always be set to True.
+ Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This argument can be used
+ in eager mode, in graph mode the value will always be set to True.
training (`bool`, *optional*, defaults to `False`):
Whether or not to use the model in training mode (some modules like dropout modules have different
behaviors between training and evaluation).
@@ -705,9 +703,8 @@ class TFMarianEncoder(tf.keras.layers.Layer):
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
provide it.
- Indices can be obtained using [`MarianTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`]
- for details.
+ Indices can be obtained using [`MarianTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -717,15 +714,16 @@ class TFMarianEncoder(tf.keras.layers.Layer):
- 0 for tokens that are **masked**.
[What are attention masks?](../glossary#attention-mask)
- head_mask (`tf.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, `optional): Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`:
+ head_mask (`tf.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, `optional):
+ Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`:
- 1 indicates the head is **not masked**,
- 0 indicates the head is **masked**.
inputs_embeds (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded
- representation. This is useful if you want more control over how to convert `input_ids` indices
- into associated vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
+ This is useful if you want more control over how to convert `input_ids` indices into associated vectors
+ than the model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under
returned tensors for more detail. This argument can be used only in eager mode, in graph mode the value
@@ -735,8 +733,8 @@ class TFMarianEncoder(tf.keras.layers.Layer):
for more detail. This argument can be used only in eager mode, in graph mode the value in the config
will be used instead.
return_dict (`bool`, *optional*):
- Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This
- argument can be used in eager mode, in graph mode the value will always be set to True.
+ Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This argument can be
+ used in eager mode, in graph mode the value will always be set to True.
training (`bool`, *optional*, defaults to `False`):
Whether or not to use the model in training mode (some modules like dropout modules have different
behaviors between training and evaluation).
@@ -876,9 +874,8 @@ class TFMarianDecoder(tf.keras.layers.Layer):
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
provide it.
- Indices can be obtained using [`MarianTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`]
- for details.
+ Indices can be obtained using [`MarianTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -915,11 +912,13 @@ class TFMarianDecoder(tf.keras.layers.Layer):
Contains precomputed key and value hidden-states of the attention blocks. Can be used to speed up
decoding.
- If `past_key_values` are used, the user can optionally input only the last
- `decoder_input_ids` (those that don't have their past key value states given to this model) of
- shape `(batch_size, 1)` instead of all ``decoder_input_ids``` of shape `(batch_size,
- sequence_length)`. inputs_embeds (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices
- into associated vectors than the model's internal embedding lookup matrix.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those
+ that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of
+ all ``decoder_input_ids``` of shape `(batch_size, sequence_length)`. inputs_embeds (`tf.Tensor` of
+ shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing
+ `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more
+ control over how to convert `input_ids` indices into associated vectors than the model's internal
+ embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under
returned tensors for more detail. This argument can be used only in eager mode, in graph mode the value
@@ -929,8 +928,8 @@ class TFMarianDecoder(tf.keras.layers.Layer):
for more detail. This argument can be used only in eager mode, in graph mode the value in the config
will be used instead.
return_dict (`bool`, *optional*):
- Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This
- argument can be used in eager mode, in graph mode the value will always be set to True.
+ Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This argument can be
+ used in eager mode, in graph mode the value will always be set to True.
training (`bool`, *optional*, defaults to `False`):
Whether or not to use the model in training mode (some modules like dropout modules have different
behaviors between training and evaluation).
@@ -1376,7 +1375,8 @@ class TFMarianMTModel(TFMarianPreTrainedModel, TFCausalLanguageModelingLoss):
):
r"""
labels (`tf.tensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
+ Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
+ config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
(masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
Returns:
diff --git a/src/transformers/models/marian/tokenization_marian.py b/src/transformers/models/marian/tokenization_marian.py
index 3ec362565c..12c06baba8 100644
--- a/src/transformers/models/marian/tokenization_marian.py
+++ b/src/transformers/models/marian/tokenization_marian.py
@@ -57,8 +57,8 @@ class MarianTokenizer(PreTrainedTokenizer):
r"""
Construct a Marian tokenizer. Based on [SentencePiece](https://github.com/google/sentencepiece).
- This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods.
- Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
+ this superclass for more information regarding those methods.
Args:
source_spm (`str`):
@@ -83,7 +83,9 @@ class MarianTokenizer(PreTrainedTokenizer):
additional_special_tokens (`List[str]`, *optional*, defaults to `["", ""]`):
Additional special tokens used by the tokenizer.
sp_model_kwargs (`dict`, *optional*):
- Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things, to set:
+ Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for
+ SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things,
+ to set:
- `enable_sampling`: Enable subword regularization.
- `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout.
diff --git a/src/transformers/models/mbart/configuration_mbart.py b/src/transformers/models/mbart/configuration_mbart.py
index f1f08cd75d..fc0775511c 100644
--- a/src/transformers/models/mbart/configuration_mbart.py
+++ b/src/transformers/models/mbart/configuration_mbart.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" MBART model configuration """
+""" MBART model configuration"""
from collections import OrderedDict
from typing import Any, Mapping, Optional
@@ -34,19 +34,19 @@ MBART_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class MBartConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`MBartModel`]. It is used to
- instantiate an MBART model according to the specified arguments, defining the model architecture. Instantiating a
- configuration with the defaults will yield a similar configuration to that of the MBART [facebook/mbart-large-cc25](https://huggingface.co/facebook/mbart-large-cc25) architecture.
+ This is the configuration class to store the configuration of a [`MBartModel`]. It is used to instantiate an MBART
+ model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
+ defaults will yield a similar configuration to that of the MBART
+ [facebook/mbart-large-cc25](https://huggingface.co/facebook/mbart-large-cc25) architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
vocab_size (`int`, *optional*, defaults to 50265):
Vocabulary size of the MBART model. Defines the number of different tokens that can be represented by the
- `inputs_ids` passed when calling [`MBartModel`] or
- [`TFMBartModel`].
+ `inputs_ids` passed when calling [`MBartModel`] or [`TFMBartModel`].
d_model (`int`, *optional*, defaults to 1024):
Dimensionality of the layers and the pooler layer.
encoder_layers (`int`, *optional*, defaults to 12):
@@ -62,8 +62,8 @@ class MBartConfig(PretrainedConfig):
encoder_ffn_dim (`int`, *optional*, defaults to 4096):
Dimensionality of the "intermediate" (often named feed-forward) layer in decoder.
activation_function (`str` or `function`, *optional*, defaults to `"gelu"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"silu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"silu"` and `"gelu_new"` are supported.
dropout (`float`, *optional*, defaults to 0.1):
The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
attention_dropout (`float`, *optional*, defaults to 0.0):
@@ -78,11 +78,11 @@ class MBartConfig(PretrainedConfig):
init_std (`float`, *optional*, defaults to 0.02):
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
encoder_layerdrop: (`float`, *optional*, defaults to 0.0):
- The LayerDrop probability for the encoder. See the [LayerDrop paper](see
- https://arxiv.org/abs/1909.11556) for more details.
+ The LayerDrop probability for the encoder. See the [LayerDrop paper](see https://arxiv.org/abs/1909.11556)
+ for more details.
decoder_layerdrop: (`float`, *optional*, defaults to 0.0):
- The LayerDrop probability for the decoder. See the [LayerDrop paper](see
- https://arxiv.org/abs/1909.11556) for more details.
+ The LayerDrop probability for the decoder. See the [LayerDrop paper](see https://arxiv.org/abs/1909.11556)
+ for more details.
scale_embedding (`bool`, *optional*, defaults to `False`):
Scale embeddings by diving by sqrt(d_model).
use_cache (`bool`, *optional*, defaults to `True`):
diff --git a/src/transformers/models/mbart/modeling_flax_mbart.py b/src/transformers/models/mbart/modeling_flax_mbart.py
index d6a425ff3c..f569d08b05 100644
--- a/src/transformers/models/mbart/modeling_flax_mbart.py
+++ b/src/transformers/models/mbart/modeling_flax_mbart.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" Flax MBart model. """
+""" Flax MBart model."""
import math
import random
@@ -59,12 +59,13 @@ _TOKENIZER_FOR_DOC = "MBartTokenizer"
MBART_START_DOCSTRING = r"""
- This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading or saving, resizing the input
- embeddings, pruning heads etc.)
+ This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a Flax Linen [flax.nn.Module](https://flax.readthedocs.io/en/latest/_autosummary/flax.nn.module.html) subclass. Use it as a regular Flax
- Module and refer to the Flax documentation for all matter related to general usage and behavior.
+ This model is also a Flax Linen
+ [flax.nn.Module](https://flax.readthedocs.io/en/latest/_autosummary/flax.nn.module.html) subclass. Use it as a
+ regular Flax Module and refer to the Flax documentation for all matter related to general usage and behavior.
Finally, this model supports inherent JAX features such as:
@@ -76,11 +77,10 @@ MBART_START_DOCSTRING = r"""
Parameters:
config ([`MBartConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the
- model weights.
+ configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the model weights.
dtype (`jax.numpy.dtype`, *optional*, defaults to `jax.numpy.float32`):
- The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on
- GPUs) and `jax.numpy.bfloat16` (on TPUs).
+ The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on GPUs) and
+ `jax.numpy.bfloat16` (on TPUs).
This can be used to enable mixed-precision training or half-precision inference on GPUs or TPUs. If
specified all the computation will be performed with the given `dtype`.
@@ -88,8 +88,8 @@ MBART_START_DOCSTRING = r"""
**Note that this only specifies the dtype of the computation and does not influence the dtype of model
parameters.**
- If you wish to change the dtype of the model parameters, see
- [`~FlaxPreTrainedModel.to_fp16`] and [`~FlaxPreTrainedModel.to_bf16`].
+ If you wish to change the dtype of the model parameters, see [`~FlaxPreTrainedModel.to_fp16`] and
+ [`~FlaxPreTrainedModel.to_bf16`].
"""
MBART_INPUTS_DOCSTRING = r"""
@@ -98,9 +98,8 @@ MBART_INPUTS_DOCSTRING = r"""
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
it.
- Indices can be obtained using [`MBartTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`MBartTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`jnp.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
@@ -113,22 +112,23 @@ MBART_INPUTS_DOCSTRING = r"""
decoder_input_ids (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`, *optional*):
Indices of decoder input sequence tokens in the vocabulary.
- Indices can be obtained using [`MBartTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`MBartTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are decoder input IDs?](../glossary#decoder-input-ids)
For translation and summarization training, `decoder_input_ids` should be provided. If no
- `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to
- the right for denoising pre-training following the paper.
+ `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right
+ for denoising pre-training following the paper.
decoder_attention_mask (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`, *optional*):
- Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will
- also be used by default.
+ Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
+ be used by default.
- If you want to change padding behavior, you should modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy.
+ If you want to change padding behavior, you should modify to your needs. See diagram 1 in [the
+ paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy.
position_ids (`numpy.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
decoder_position_ids (`numpy.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the
range `[0, config.max_position_embeddings - 1]`.
@@ -149,9 +149,8 @@ MBART_ENCODE_INPUTS_DOCSTRING = r"""
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
it.
- Indices can be obtained using [`MBartTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`MBartTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`jnp.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
@@ -162,7 +161,8 @@ MBART_ENCODE_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
position_ids (`numpy.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -178,20 +178,18 @@ MBART_DECODE_INPUTS_DOCSTRING = r"""
decoder_input_ids (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`):
Indices of decoder input sequence tokens in the vocabulary.
- Indices can be obtained using [`MBartTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`MBartTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are decoder input IDs?](../glossary#decoder-input-ids)
For translation and summarization training, `decoder_input_ids` should be provided. If no
- `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to
- the right for denoising pre-training following the paper.
+ `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right
+ for denoising pre-training following the paper.
encoder_outputs (`tuple(tuple(jnp.ndarray)`):
- Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*:
- `attentions`) `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`,
- *optional*) is a sequence of hidden-states at the output of the last layer of the encoder. Used in the
- cross-attention of the decoder.
+ Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`)
+ `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of
+ hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder.
encoder_attention_mask (`jnp.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
@@ -200,10 +198,11 @@ MBART_DECODE_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
decoder_attention_mask (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`, *optional*):
- Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will
- also be used by default.
+ Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
+ be used by default.
- If you want to change padding behavior, you should modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy.
+ If you want to change padding behavior, you should modify to your needs. See diagram 1 in [the
+ paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy.
decoder_position_ids (`numpy.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the
range `[0, config.max_position_embeddings - 1]`.
@@ -993,9 +992,10 @@ class FlaxMBartPreTrainedModel(FlaxPreTrainedModel):
maximum possible length for auto-regressive decoding. Defines the sequence length of the initialized
cache.
encoder_outputs (`Union[FlaxBaseModelOutput, tuple(tuple(jnp.ndarray)]`):
- `encoder_outputs` consists of (`last_hidden_state`, *optional*: `hidden_states`,
- *optional*: `attentions`). `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of hidden-states at the output of the last layer of the
- encoder. Used in the cross-attention of the decoder.
+ `encoder_outputs` consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*:
+ `attentions`). `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*)
+ is a sequence of hidden-states at the output of the last layer of the encoder. Used in the
+ cross-attention of the decoder.
"""
# init input variables to retrieve cache
decoder_input_ids = jnp.ones((batch_size, max_length), dtype="i4")
@@ -1534,30 +1534,28 @@ FLAX_MBART_CONDITIONAL_GENERATION_DOCSTRING = r"""
>>> from transformers import MBartTokenizer, FlaxMBartForConditionalGeneration, MBartConfig
- >>> model = FlaxMBartForConditionalGeneration.from_pretrained('facebook/mbart-large-cc25')
- >>> tokenizer = MBartTokenizer.from_pretrained('facebook/mbart-large-cc25')
+ >>> model = FlaxMBartForConditionalGeneration.from_pretrained('facebook/mbart-large-cc25') >>> tokenizer =
+ MBartTokenizer.from_pretrained('facebook/mbart-large-cc25')
- >>> ARTICLE_TO_SUMMARIZE = "Meine Freunde sind cool, aber sie essen zu viel Kuchen."
- >>> inputs = tokenizer([ARTICLE_TO_SUMMARIZE], max_length=1024, return_tensors='np')
+ >>> ARTICLE_TO_SUMMARIZE = "Meine Freunde sind cool, aber sie essen zu viel Kuchen." >>> inputs =
+ tokenizer([ARTICLE_TO_SUMMARIZE], max_length=1024, return_tensors='np')
- >>> # Generate Summary
- >>> summary_ids = model.generate(inputs['input_ids'], num_beams=4, max_length=5, early_stopping=True).sequences
- >>> print([tokenizer.decode(g, skip_special_tokens=True, clean_up_tokenization_spaces=False) for g in summary_ids])
+ >>> # Generate Summary >>> summary_ids = model.generate(inputs['input_ids'], num_beams=4, max_length=5,
+ early_stopping=True).sequences >>> print([tokenizer.decode(g, skip_special_tokens=True,
+ clean_up_tokenization_spaces=False) for g in summary_ids])
Mask filling example::
- >>> from transformers import MBartTokenizer, FlaxMBartForConditionalGeneration
- >>> tokenizer = MBartTokenizer.from_pretrained('facebook/mbart-large-cc25')
- >>> # de_DE is the language symbol id for German
- >>> TXT = " Meine Freunde sind nett aber sie essen zu viel Kuchen. de_DE"
+ >>> from transformers import MBartTokenizer, FlaxMBartForConditionalGeneration >>> tokenizer =
+ MBartTokenizer.from_pretrained('facebook/mbart-large-cc25') >>> # de_DE is the language symbol id for
+ German >>> TXT = " Meine Freunde sind nett aber sie essen zu viel Kuchen. de_DE"
- >>> model = FlaxMBartForConditionalGeneration.from_pretrained('facebook/mbart-large-cc25')
- >>> input_ids = tokenizer([TXT], add_special_tokens=False, return_tensors='np')['input_ids']
- >>> logits = model(input_ids).logits
+ >>> model = FlaxMBartForConditionalGeneration.from_pretrained('facebook/mbart-large-cc25') >>> input_ids =
+ tokenizer([TXT], add_special_tokens=False, return_tensors='np')['input_ids'] >>> logits =
+ model(input_ids).logits
- >>> masked_index = (input_ids[0] == tokenizer.mask_token_id).nonzero()[0].item()
- >>> probs = logits[0, masked_index].softmax(dim=0)
- >>> values, predictions = probs.topk(5)
+ >>> masked_index = (input_ids[0] == tokenizer.mask_token_id).nonzero()[0].item() >>> probs = logits[0,
+ masked_index].softmax(dim=0) >>> values, predictions = probs.topk(5)
>>> tokenizer.decode(predictions).split()
"""
diff --git a/src/transformers/models/mbart/modeling_mbart.py b/src/transformers/models/mbart/modeling_mbart.py
index 0892b62b58..73bc4f0ef8 100755
--- a/src/transformers/models/mbart/modeling_mbart.py
+++ b/src/transformers/models/mbart/modeling_mbart.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" PyTorch MBART model. """
+""" PyTorch MBART model."""
import copy
import math
import random
@@ -386,7 +386,8 @@ class MBartDecoderLayer(nn.Module):
hidden_states (`torch.FloatTensor`): input to the layer of shape *(seq_len, batch, embed_dim)*
attention_mask (`torch.FloatTensor`): attention mask of size
*(batch, 1, tgt_len, src_len)* where padding elements are indicated by very large negative values.
- encoder_hidden_states (`torch.FloatTensor`): cross attention input to the layer of shape *(seq_len, batch, embed_dim)*
+ encoder_hidden_states (`torch.FloatTensor`):
+ cross attention input to the layer of shape *(seq_len, batch, embed_dim)*
encoder_attention_mask (`torch.FloatTensor`): encoder attention mask of size
*(batch, 1, tgt_len, src_len)* where padding elements are indicated by very large negative values.
layer_head_mask (`torch.FloatTensor`): mask for attention heads in a given layer of size
@@ -515,13 +516,13 @@ class MBartPreTrainedModel(PreTrainedModel):
MBART_START_DOCSTRING = r"""
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
Parameters:
config ([`MBartConfig`]):
@@ -535,30 +536,28 @@ MBART_GENERATION_EXAMPLE = r"""
>>> from transformers import MBartTokenizer, MBartForConditionalGeneration, MBartConfig
- >>> model = MBartForConditionalGeneration.from_pretrained('facebook/mbart-large-cc25')
- >>> tokenizer = MBartTokenizer.from_pretrained('facebook/mbart-large-cc25')
+ >>> model = MBartForConditionalGeneration.from_pretrained('facebook/mbart-large-cc25') >>> tokenizer =
+ MBartTokenizer.from_pretrained('facebook/mbart-large-cc25')
- >>> ARTICLE_TO_SUMMARIZE = "Meine Freunde sind cool, aber sie essen zu viel Kuchen."
- >>> inputs = tokenizer([ARTICLE_TO_SUMMARIZE], max_length=1024, return_tensors='pt')
+ >>> ARTICLE_TO_SUMMARIZE = "Meine Freunde sind cool, aber sie essen zu viel Kuchen." >>> inputs =
+ tokenizer([ARTICLE_TO_SUMMARIZE], max_length=1024, return_tensors='pt')
- >>> # Generate Summary
- >>> summary_ids = model.generate(inputs['input_ids'], num_beams=4, max_length=5, early_stopping=True)
- >>> print([tokenizer.decode(g, skip_special_tokens=True, clean_up_tokenization_spaces=False) for g in summary_ids])
+ >>> # Generate Summary >>> summary_ids = model.generate(inputs['input_ids'], num_beams=4, max_length=5,
+ early_stopping=True) >>> print([tokenizer.decode(g, skip_special_tokens=True,
+ clean_up_tokenization_spaces=False) for g in summary_ids])
Mask filling example::
- >>> from transformers import MBartTokenizer, MBartForConditionalGeneration
- >>> tokenizer = MBartTokenizer.from_pretrained('facebook/mbart-large-cc25')
- >>> # de_DE is the language symbol id for German
- >>> TXT = " Meine Freunde sind nett aber sie essen zu viel Kuchen. de_DE"
+ >>> from transformers import MBartTokenizer, MBartForConditionalGeneration >>> tokenizer =
+ MBartTokenizer.from_pretrained('facebook/mbart-large-cc25') >>> # de_DE is the language symbol id for
+ German >>> TXT = " Meine Freunde sind nett aber sie essen zu viel Kuchen. de_DE"
- >>> model = MBartForConditionalGeneration.from_pretrained('facebook/mbart-large-cc25')
- >>> input_ids = tokenizer([TXT], add_special_tokens=False, return_tensors='pt')['input_ids']
- >>> logits = model(input_ids).logits
+ >>> model = MBartForConditionalGeneration.from_pretrained('facebook/mbart-large-cc25') >>> input_ids =
+ tokenizer([TXT], add_special_tokens=False, return_tensors='pt')['input_ids'] >>> logits =
+ model(input_ids).logits
- >>> masked_index = (input_ids[0] == tokenizer.mask_token_id).nonzero().item()
- >>> probs = logits[0, masked_index].softmax(dim=0)
- >>> values, predictions = probs.topk(5)
+ >>> masked_index = (input_ids[0] == tokenizer.mask_token_id).nonzero().item() >>> probs = logits[0,
+ masked_index].softmax(dim=0) >>> values, predictions = probs.topk(5)
>>> tokenizer.decode(predictions).split()
"""
@@ -569,9 +568,8 @@ MBART_INPUTS_DOCSTRING = r"""
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
it.
- Indices can be obtained using [`MBartTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`MBartTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -584,9 +582,8 @@ MBART_INPUTS_DOCSTRING = r"""
decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
Indices of decoder input sequence tokens in the vocabulary.
- Indices can be obtained using [`MBartTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`MBartTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are decoder input IDs?](../glossary#decoder-input-ids)
@@ -596,11 +593,11 @@ MBART_INPUTS_DOCSTRING = r"""
`past_key_values`).
For translation and summarization training, `decoder_input_ids` should be provided. If no
- `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to
- the right for denoising pre-training following the paper.
+ `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right
+ for denoising pre-training following the paper.
decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
- Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will
- also be used by default.
+ Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
+ be used by default.
head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*):
Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`:
@@ -614,39 +611,42 @@ MBART_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*):
- Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, 1]`:
+ Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0,
+ 1]`:
- 1 indicates the head is **not masked**,
- 0 indicates the head is **masked**.
encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*):
- Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*:
- `attentions`) `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`,
- *optional*) is a sequence of hidden-states at the output of the last layer of the encoder. Used in the
- cross-attention of the decoder.
+ Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`)
+ `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of
+ hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder.
past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors
- of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
- shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
+ Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
+ `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape
+ `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all ``decoder_input_ids``` of shape `(batch_size, sequence_length)`. inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ ``decoder_input_ids``` of shape `(batch_size, sequence_length)`. inputs_embeds (`torch.FloatTensor` of
+ shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids`
+ you can choose to directly pass an embedded representation. This is useful if you want more control over
+ how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup
+ matrix.
decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*):
Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded
- representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds`
- have to be input (see `past_key_values`). This is useful if you want more control over how to convert
+ representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be
+ input (see `past_key_values`). This is useful if you want more control over how to convert
`decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix.
- If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds`
- takes the value of `inputs_embeds`.
+ If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds` takes the value
+ of `inputs_embeds`.
use_cache (`bool`, *optional*):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`).
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`).
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -717,9 +717,8 @@ class MBartEncoder(MBartPreTrainedModel):
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
provide it.
- Indices can be obtained using [`MBartTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`]
- for details.
+ Indices can be obtained using [`MBartTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -736,9 +735,9 @@ class MBartEncoder(MBartPreTrainedModel):
- 0 indicates the head is **masked**.
inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded
- representation. This is useful if you want more control over how to convert `input_ids` indices
- into associated vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
+ This is useful if you want more control over how to convert `input_ids` indices into associated vectors
+ than the model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under
returned tensors for more detail.
@@ -914,9 +913,8 @@ class MBartDecoder(MBartPreTrainedModel):
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
provide it.
- Indices can be obtained using [`MBartTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`]
- for details.
+ Indices can be obtained using [`MBartTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -951,19 +949,20 @@ class MBartDecoder(MBartPreTrainedModel):
- 0 indicates the head is **masked**.
past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2
- tensors of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional
- tensors of shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
+ Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of
+ shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
+ shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
Contains pre-computed hidden-states (key and values in the self-attention blocks and in the
- cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential
- decoding.
+ cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
- If `past_key_values` are used, the user can optionally input only the last
- `decoder_input_ids` (those that don't have their past key value states given to this model) of
- shape `(batch_size, 1)` instead of all ``decoder_input_ids``` of shape `(batch_size,
- sequence_length)`. inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices
- into associated vectors than the model's internal embedding lookup matrix.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those
+ that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of
+ all ``decoder_input_ids``` of shape `(batch_size, sequence_length)`. inputs_embeds (`torch.FloatTensor`
+ of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing
+ `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more
+ control over how to convert `input_ids` indices into associated vectors than the model's internal
+ embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under
returned tensors for more detail.
@@ -1297,7 +1296,8 @@ class MBartForConditionalGeneration(MBartPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
+ Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
+ config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
(masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
Returns:
@@ -1439,7 +1439,8 @@ class MBartForSequenceClassification(MBartPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
if labels is not None:
@@ -1708,9 +1709,8 @@ class MBartForCausalLM(MBartPreTrainedModel):
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
provide it.
- Indices can be obtained using [`MBartTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`]
- for details.
+ Indices can be obtained using [`MBartTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -1739,25 +1739,24 @@ class MBartForCausalLM(MBartPreTrainedModel):
- 0 indicates the head is **masked**.
past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2
- tensors of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional
- tensors of shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. The two
- additional tensors are only required when the model is used as a decoder in a Sequence to Sequence
- model.
+ Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of
+ shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
+ shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. The two additional
+ tensors are only required when the model is used as a decoder in a Sequence to Sequence model.
Contains pre-computed hidden-states (key and values in the self-attention blocks and in the
- cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential
- decoding.
+ cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those
+ that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of
+ all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are
- ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
+ Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
+ config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
+ (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
use_cache (`bool`, *optional*):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`).
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
+ (see `past_key_values`).
- 1 for tokens that are **not masked**,
- 0 for tokens that are **masked**.
diff --git a/src/transformers/models/mbart/modeling_tf_mbart.py b/src/transformers/models/mbart/modeling_tf_mbart.py
index 0702b8d178..972a441837 100644
--- a/src/transformers/models/mbart/modeling_tf_mbart.py
+++ b/src/transformers/models/mbart/modeling_tf_mbart.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" TF 2.0 MBart model. """
+""" TF 2.0 MBart model."""
import random
@@ -377,7 +377,8 @@ class TFMBartDecoderLayer(tf.keras.layers.Layer):
hidden_states (`tf.Tensor`): input to the layer of shape *(seq_len, batch, embed_dim)*
attention_mask (`tf.Tensor`): attention mask of size
*(batch, 1, tgt_len, src_len)* where padding elements are indicated by very large negative values.
- encoder_hidden_states (`tf.Tensor`): cross attention input to the layer of shape *(seq_len, batch, embed_dim)*
+ encoder_hidden_states (`tf.Tensor`):
+ cross attention input to the layer of shape *(seq_len, batch, embed_dim)*
encoder_attention_mask (`tf.Tensor`): encoder attention mask of size
*(batch, 1, tgt_len, src_len)* where padding elements are indicated by very large negative values.
layer_head_mask (`tf.Tensor`): mask for attention heads in a given layer of size
@@ -475,13 +476,13 @@ class TFMBartPreTrainedModel(TFPreTrainedModel):
MBART_START_DOCSTRING = r"""
- This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading or saving, resizing the input
- embeddings, pruning heads etc.)
+ This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use
- it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage
- and behavior.
+ This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it
+ as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and
+ behavior.
@@ -490,11 +491,11 @@ MBART_START_DOCSTRING = r"""
- having all inputs as keyword arguments (like PyTorch models), or
- having all inputs as a list, tuple or dict in the first positional arguments.
- This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all
- the tensors in the first argument of the model call function: `model(inputs)`.
+ This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all the
+ tensors in the first argument of the model call function: `model(inputs)`.
- If you choose this second option, there are three possibilities you can use to gather all the input Tensors in
- the first positional argument :
+ If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the
+ first positional argument :
- a single Tensor with `input_ids` only and nothing else: `model(input_ids)`
- a list of varying length with one or several input Tensors IN THE ORDER given in the docstring:
@@ -507,8 +508,7 @@ MBART_START_DOCSTRING = r"""
Args:
config ([`MBartConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~TFPreTrainedModel.from_pretrained`] method to load the
- model weights.
+ configuration. Check out the [`~TFPreTrainedModel.from_pretrained`] method to load the model weights.
"""
MBART_INPUTS_DOCSTRING = r"""
@@ -516,9 +516,8 @@ MBART_INPUTS_DOCSTRING = r"""
input_ids (`tf.Tensor` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`MBartTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`MBartTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`tf.Tensor` of shape `({0})`, *optional*):
@@ -531,9 +530,8 @@ MBART_INPUTS_DOCSTRING = r"""
decoder_input_ids (`tf.Tensor` of shape `(batch_size, target_sequence_length)`, *optional*):
Indices of decoder input sequence tokens in the vocabulary.
- Indices can be obtained using [`MBartTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`MBartTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are decoder input IDs?](../glossary#decoder-input-ids)
@@ -543,8 +541,8 @@ MBART_INPUTS_DOCSTRING = r"""
`past_key_values`).
For translation and summarization training, `decoder_input_ids` should be provided. If no
- `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to
- the right for denoising pre-training following the paper.
+ `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right
+ for denoising pre-training following the paper.
decoder_attention_mask (`tf.Tensor` of shape `(batch_size, target_sequence_length)`, *optional*):
will be made by default and ignore pad tokens. It is not recommended to set this for most use cases.
head_mask (`tf.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*):
@@ -570,12 +568,12 @@ MBART_INPUTS_DOCSTRING = r"""
of shape `(batch_size, sequence_length, hidden_size)` is a sequence of
past_key_values (`Tuple[Tuple[tf.Tensor]]` of length `config.n_layers`)
contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ `decoder_input_ids` of shape `(batch_size, sequence_length)`.
use_cache (`bool`, *optional*, defaults to `True`):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`). Set to `False` during training, `True` during generation
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`). Set to `False` during training, `True` during generation
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the
@@ -585,8 +583,8 @@ MBART_INPUTS_DOCSTRING = r"""
more detail. This argument can be used only in eager mode, in graph mode the value in the config will be
used instead.
return_dict (`bool`, *optional*):
- Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This
- argument can be used in eager mode, in graph mode the value will always be set to True.
+ Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This argument can be used
+ in eager mode, in graph mode the value will always be set to True.
training (`bool`, *optional*, defaults to `False`):
Whether or not to use the model in training mode (some modules like dropout modules have different
behaviors between training and evaluation).
@@ -597,28 +595,25 @@ MBART_GENERATION_EXAMPLE = r"""
>>> from transformers import MBartTokenizer, TFMBartForConditionalGeneration, MBartConfig
- >>> model = MBartForConditionalGeneration.from_pretrained('facebook/mbart-large-cc25')
- >>> tokenizer = MBartTokenizer.from_pretrained('facebook/mbart-large-cc25')
+ >>> model = MBartForConditionalGeneration.from_pretrained('facebook/mbart-large-cc25') >>> tokenizer =
+ MBartTokenizer.from_pretrained('facebook/mbart-large-cc25')
- >>> ARTICLE_TO_SUMMARIZE = "Meine Freunde sind cool, aber sie essen zu viel Kuchen."
- >>> inputs = tokenizer([ARTICLE_TO_SUMMARIZE], max_length=1024, return_tensors='tf')
+ >>> ARTICLE_TO_SUMMARIZE = "Meine Freunde sind cool, aber sie essen zu viel Kuchen." >>> inputs =
+ tokenizer([ARTICLE_TO_SUMMARIZE], max_length=1024, return_tensors='tf')
- >>> # Generate Summary
- >>> summary_ids = model.generate(inputs['input_ids'], num_beams=4, max_length=5, early_stopping=True)
- >>> print([tokenizer.decode(g, skip_special_tokens=True, clean_up_tokenization_spaces=False) for g in summary_ids])
+ >>> # Generate Summary >>> summary_ids = model.generate(inputs['input_ids'], num_beams=4, max_length=5,
+ early_stopping=True) >>> print([tokenizer.decode(g, skip_special_tokens=True,
+ clean_up_tokenization_spaces=False) for g in summary_ids])
Mask filling example::
- >>> from transformers import MBartTokenizer, TFMBartForConditionalGeneration
- >>> tokenizer = MBartTokenizer.from_pretrained('facebook/mbart-large-cc25')
- >>> # de_DE is the language symbol id for German
- >>> TXT = " Meine Freunde sind nett aber sie essen zu viel Kuchen. de_DE"
+ >>> from transformers import MBartTokenizer, TFMBartForConditionalGeneration >>> tokenizer =
+ MBartTokenizer.from_pretrained('facebook/mbart-large-cc25') >>> # de_DE is the language symbol id for
+ German >>> TXT = " Meine Freunde sind nett aber sie essen zu viel Kuchen. de_DE"
- >>> model = MBartForConditionalGeneration.from_pretrained('facebook/mbart-large-cc25')
- >>> input_ids = tokenizer([TXT], add_special_tokens=False, return_tensors='tf')['input_ids']
- >>> logits = model(input_ids).logits
- >>> probs = tf.nn.softmax(logits[0])
- >>> # probs[5] is associated with the mask token
+ >>> model = MBartForConditionalGeneration.from_pretrained('facebook/mbart-large-cc25') >>> input_ids =
+ tokenizer([TXT], add_special_tokens=False, return_tensors='tf')['input_ids'] >>> logits =
+ model(input_ids).logits >>> probs = tf.nn.softmax(logits[0]) >>> # probs[5] is associated with the mask token
"""
@@ -676,9 +671,8 @@ class TFMBartEncoder(tf.keras.layers.Layer):
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
provide it.
- Indices can be obtained using [`MBartTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`]
- for details.
+ Indices can be obtained using [`MBartTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -688,15 +682,16 @@ class TFMBartEncoder(tf.keras.layers.Layer):
- 0 for tokens that are **masked**.
[What are attention masks?](../glossary#attention-mask)
- head_mask (`tf.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, `optional): Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`:
+ head_mask (`tf.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, `optional):
+ Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`:
- 1 indicates the head is **not masked**,
- 0 indicates the head is **masked**.
inputs_embeds (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded
- representation. This is useful if you want more control over how to convert `input_ids` indices
- into associated vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
+ This is useful if you want more control over how to convert `input_ids` indices into associated vectors
+ than the model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under
returned tensors for more detail. This argument can be used only in eager mode, in graph mode the value
@@ -706,8 +701,8 @@ class TFMBartEncoder(tf.keras.layers.Layer):
for more detail. This argument can be used only in eager mode, in graph mode the value in the config
will be used instead.
return_dict (`bool`, *optional*):
- Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This
- argument can be used in eager mode, in graph mode the value will always be set to True.
+ Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This argument can be
+ used in eager mode, in graph mode the value will always be set to True.
training (`bool`, *optional*, defaults to `False`):
Whether or not to use the model in training mode (some modules like dropout modules have different
behaviors between training and evaluation).
@@ -852,9 +847,8 @@ class TFMBartDecoder(tf.keras.layers.Layer):
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
provide it.
- Indices can be obtained using [`MBartTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`]
- for details.
+ Indices can be obtained using [`MBartTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -891,11 +885,13 @@ class TFMBartDecoder(tf.keras.layers.Layer):
Contains precomputed key and value hidden-states of the attention blocks. Can be used to speed up
decoding.
- If `past_key_values` are used, the user can optionally input only the last
- `decoder_input_ids` (those that don't have their past key value states given to this model) of
- shape `(batch_size, 1)` instead of all ``decoder_input_ids``` of shape `(batch_size,
- sequence_length)`. inputs_embeds (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices
- into associated vectors than the model's internal embedding lookup matrix.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those
+ that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of
+ all ``decoder_input_ids``` of shape `(batch_size, sequence_length)`. inputs_embeds (`tf.Tensor` of
+ shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing
+ `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more
+ control over how to convert `input_ids` indices into associated vectors than the model's internal
+ embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under
returned tensors for more detail. This argument can be used only in eager mode, in graph mode the value
@@ -905,8 +901,8 @@ class TFMBartDecoder(tf.keras.layers.Layer):
for more detail. This argument can be used only in eager mode, in graph mode the value in the config
will be used instead.
return_dict (`bool`, *optional*):
- Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This
- argument can be used in eager mode, in graph mode the value will always be set to True.
+ Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This argument can be
+ used in eager mode, in graph mode the value will always be set to True.
training (`bool`, *optional*, defaults to `False`):
Whether or not to use the model in training mode (some modules like dropout modules have different
behaviors between training and evaluation).
@@ -1358,7 +1354,8 @@ class TFMBartForConditionalGeneration(TFMBartPreTrainedModel, TFCausalLanguageMo
):
"""
labels (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
+ Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
+ config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
(masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
Returns:
diff --git a/src/transformers/models/mbart/tokenization_mbart.py b/src/transformers/models/mbart/tokenization_mbart.py
index ce33736f74..23443dc238 100644
--- a/src/transformers/models/mbart/tokenization_mbart.py
+++ b/src/transformers/models/mbart/tokenization_mbart.py
@@ -71,9 +71,8 @@ class MBartTokenizer(XLMRobertaTokenizer):
"""
Construct an MBART tokenizer.
- [`MBartTokenizer`] is a subclass of [`XLMRobertaTokenizer`]. Refer to
- superclass [`XLMRobertaTokenizer`] for usage examples and documentation concerning the
- initialization parameters and other methods.
+ [`MBartTokenizer`] is a subclass of [`XLMRobertaTokenizer`]. Refer to superclass [`XLMRobertaTokenizer`] for usage
+ examples and documentation concerning the initialization parameters and other methods.
The tokenization method is ` ` for source language documents, and ``
``` for target language documents.
diff --git a/src/transformers/models/mbart/tokenization_mbart_fast.py b/src/transformers/models/mbart/tokenization_mbart_fast.py
index bf578bb2c4..b6850187bc 100644
--- a/src/transformers/models/mbart/tokenization_mbart_fast.py
+++ b/src/transformers/models/mbart/tokenization_mbart_fast.py
@@ -82,11 +82,12 @@ FAIRSEQ_LANGUAGE_CODES = [
class MBartTokenizerFast(XLMRobertaTokenizerFast):
"""
- Construct a "fast" MBART tokenizer (backed by HuggingFace's *tokenizers* library). Based on [BPE](https://huggingface.co/docs/tokenizers/python/latest/components.html?highlight=BPE#models).
+ Construct a "fast" MBART tokenizer (backed by HuggingFace's *tokenizers* library). Based on
+ [BPE](https://huggingface.co/docs/tokenizers/python/latest/components.html?highlight=BPE#models).
- [`MBartTokenizerFast`] is a subclass of [`XLMRobertaTokenizerFast`]. Refer to
- superclass [`XLMRobertaTokenizerFast`] for usage examples and documentation concerning the
- initialization parameters and other methods.
+ [`MBartTokenizerFast`] is a subclass of [`XLMRobertaTokenizerFast`]. Refer to superclass
+ [`XLMRobertaTokenizerFast`] for usage examples and documentation concerning the initialization parameters and other
+ methods.
The tokenization method is ` ` for source language documents, and ``
``` for target language documents.
diff --git a/src/transformers/models/mbart50/tokenization_mbart50.py b/src/transformers/models/mbart50/tokenization_mbart50.py
index 48f34cd9ac..389535fc30 100644
--- a/src/transformers/models/mbart50/tokenization_mbart50.py
+++ b/src/transformers/models/mbart50/tokenization_mbart50.py
@@ -49,8 +49,8 @@ class MBart50Tokenizer(PreTrainedTokenizer):
"""
Construct a MBart50 tokenizer. Based on [SentencePiece](https://github.com/google/sentencepiece).
- This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods.
- Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
+ this superclass for more information regarding those methods.
Args:
vocab_file (`str`):
@@ -77,7 +77,9 @@ class MBart50Tokenizer(PreTrainedTokenizer):
The token used for masking values. This is the token used when training this model with masked language
modeling. This is the token which the model will try to predict.
sp_model_kwargs (`dict`, *optional*):
- Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things, to set:
+ Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for
+ SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things,
+ to set:
- `enable_sampling`: Enable subword regularization.
- `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout.
diff --git a/src/transformers/models/mbart50/tokenization_mbart50_fast.py b/src/transformers/models/mbart50/tokenization_mbart50_fast.py
index 7b481f4362..40448bdcbc 100644
--- a/src/transformers/models/mbart50/tokenization_mbart50_fast.py
+++ b/src/transformers/models/mbart50/tokenization_mbart50_fast.py
@@ -56,10 +56,11 @@ FAIRSEQ_LANGUAGE_CODES = ["ar_AR", "cs_CZ", "de_DE", "en_XX", "es_XX", "et_EE",
class MBart50TokenizerFast(PreTrainedTokenizerFast):
"""
- Construct a "fast" MBART tokenizer for mBART-50 (backed by HuggingFace's *tokenizers* library). Based on [BPE](https://huggingface.co/docs/tokenizers/python/latest/components.html?highlight=BPE#models).
+ Construct a "fast" MBART tokenizer for mBART-50 (backed by HuggingFace's *tokenizers* library). Based on
+ [BPE](https://huggingface.co/docs/tokenizers/python/latest/components.html?highlight=BPE#models).
- This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main
- methods. Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should
+ refer to this superclass for more information regarding those methods.
Args:
vocab_file (`str`):
diff --git a/src/transformers/models/megatron_bert/configuration_megatron_bert.py b/src/transformers/models/megatron_bert/configuration_megatron_bert.py
index 0210f0466f..4da9e0731c 100644
--- a/src/transformers/models/megatron_bert/configuration_megatron_bert.py
+++ b/src/transformers/models/megatron_bert/configuration_megatron_bert.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" MEGATRON_BERT model configuration """
+""" MEGATRON_BERT model configuration"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging
@@ -27,13 +27,13 @@ MEGATRON_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class MegatronBertConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`MegatronBertModel`]. It is
- used to instantiate a MEGATRON_BERT model according to the specified arguments, defining the model architecture.
- Instantiating a configuration with the defaults will yield a similar configuration to that of the MEGATRON_BERT
+ This is the configuration class to store the configuration of a [`MegatronBertModel`]. It is used to instantiate a
+ MEGATRON_BERT model according to the specified arguments, defining the model architecture. Instantiating a
+ configuration with the defaults will yield a similar configuration to that of the MEGATRON_BERT
[megatron-bert-uncased-345m](https://huggingface.co/nvidia/megatron-bert-uncased-345m) architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
@@ -49,8 +49,8 @@ class MegatronBertConfig(PretrainedConfig):
intermediate_size (`int`, *optional*, defaults to 4096):
Dimensionality of the "intermediate" (often named feed-forward) layer in the Transformer encoder.
hidden_act (`str` or `Callable`, *optional*, defaults to `"gelu"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"silu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"silu"` and `"gelu_new"` are supported.
hidden_dropout_prob (`float`, *optional*, defaults to 0.1):
The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
attention_probs_dropout_prob (`float`, *optional*, defaults to 0.1):
@@ -59,17 +59,17 @@ class MegatronBertConfig(PretrainedConfig):
The maximum sequence length that this model might ever be used with. Typically set this to something large
just in case (e.g., 512 or 1024 or 2048).
type_vocab_size (`int`, *optional*, defaults to 2):
- The vocabulary size of the `token_type_ids` passed when calling
- [`MegatronBertModel`].
+ The vocabulary size of the `token_type_ids` passed when calling [`MegatronBertModel`].
initializer_range (`float`, *optional*, defaults to 0.02):
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
layer_norm_eps (`float`, *optional*, defaults to 1e-12):
The epsilon used by the layer normalization layers.
position_embedding_type (`str`, *optional*, defaults to `"absolute"`):
- Type of position embedding. Choose one of `"absolute"`, `"relative_key"`,
- `"relative_key_query"`. For positional embeddings use `"absolute"`. For more information on
- `"relative_key"`, please refer to [Self-Attention with Relative Position Representations (Shaw et al.)](https://arxiv.org/abs/1803.02155). For more information on `"relative_key_query"`, please refer to
- *Method 4* in [Improve Transformer Models with Better Relative Position Embeddings (Huang et al.)](https://arxiv.org/abs/2009.13658).
+ Type of position embedding. Choose one of `"absolute"`, `"relative_key"`, `"relative_key_query"`. For
+ positional embeddings use `"absolute"`. For more information on `"relative_key"`, please refer to
+ [Self-Attention with Relative Position Representations (Shaw et al.)](https://arxiv.org/abs/1803.02155).
+ For more information on `"relative_key_query"`, please refer to *Method 4* in [Improve Transformer Models
+ with Better Relative Position Embeddings (Huang et al.)](https://arxiv.org/abs/2009.13658).
use_cache (`bool`, *optional*, defaults to `True`):
Whether or not the model should return the last key/values attentions (not used by all models). Only
relevant if `config.is_decoder=True`.
diff --git a/src/transformers/models/megatron_bert/modeling_megatron_bert.py b/src/transformers/models/megatron_bert/modeling_megatron_bert.py
index 186541e3e6..8f71a84a3f 100755
--- a/src/transformers/models/megatron_bert/modeling_megatron_bert.py
+++ b/src/transformers/models/megatron_bert/modeling_megatron_bert.py
@@ -13,7 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" PyTorch MegatronBERT model. """
+""" PyTorch MegatronBERT model."""
import math
@@ -764,19 +764,18 @@ class MegatronBertForPreTrainingOutput(ModelOutput):
MEGATRON_BERT_START_DOCSTRING = r"""
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
Parameters:
config ([`MegatronBertConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
MEGATRON_BERT_INPUTS_DOCSTRING = r"""
@@ -784,9 +783,8 @@ MEGATRON_BERT_INPUTS_DOCSTRING = r"""
input_ids (`torch.LongTensor` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`BertTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`BertTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
@@ -797,14 +795,16 @@ MEGATRON_BERT_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
@@ -814,9 +814,9 @@ MEGATRON_BERT_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -840,10 +840,9 @@ class MegatronBertModel(MegatronBertPreTrainedModel):
all you need](https://arxiv.org/abs/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit,
Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin.
- To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration
- set to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder`
- argument and `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an
- input to the forward pass.
+ To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set
+ to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and
+ `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass.
"""
def __init__(self, config, add_pooling_layer=True):
@@ -908,12 +907,12 @@ class MegatronBertModel(MegatronBertPreTrainedModel):
past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ `decoder_input_ids` of shape `(batch_size, sequence_length)`.
use_cache (`bool`, *optional*):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`).
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`).
"""
output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
output_hidden_states = (
@@ -1044,8 +1043,9 @@ class MegatronBertForPreTraining(MegatronBertPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
next_sentence_label (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair
(see `input_ids` docstring) Indices should be in `[0, 1]`:
@@ -1110,7 +1110,7 @@ class MegatronBertForPreTraining(MegatronBertPreTrainedModel):
@add_start_docstrings(
- """MegatronBert Model with a `language modeling` head on top for CLM fine-tuning. """,
+ """MegatronBert Model with a `language modeling` head on top for CLM fine-tuning.""",
MEGATRON_BERT_START_DOCSTRING,
)
class MegatronBertForCausalLM(MegatronBertPreTrainedModel):
@@ -1172,12 +1172,12 @@ class MegatronBertForCausalLM(MegatronBertPreTrainedModel):
past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ `decoder_input_ids` of shape `(batch_size, sequence_length)`.
use_cache (`bool`, *optional*):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`).
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`).
Returns:
@@ -1258,7 +1258,7 @@ class MegatronBertForCausalLM(MegatronBertPreTrainedModel):
return reordered_past
-@add_start_docstrings("""MegatronBert Model with a `language modeling` head on top. """, MEGATRON_BERT_START_DOCSTRING)
+@add_start_docstrings("""MegatronBert Model with a `language modeling` head on top.""", MEGATRON_BERT_START_DOCSTRING)
class MegatronBertForMaskedLM(MegatronBertPreTrainedModel):
_keys_to_ignore_on_load_unexpected = [r"pooler", r"seq_relationship"]
@@ -1309,8 +1309,9 @@ class MegatronBertForMaskedLM(MegatronBertPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1364,7 +1365,7 @@ class MegatronBertForMaskedLM(MegatronBertPreTrainedModel):
@add_start_docstrings(
- """MegatronBert Model with a `next sentence prediction (classification)` head on top. """,
+ """MegatronBert Model with a `next sentence prediction (classification)` head on top.""",
MEGATRON_BERT_START_DOCSTRING,
)
class MegatronBertForNextSentencePrediction(MegatronBertPreTrainedModel):
@@ -1507,8 +1508,9 @@ class MegatronBertForSequenceClassification(MegatronBertPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1605,7 +1607,8 @@ class MegatronBertForMultipleChoice(MegatronBertPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
+ Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
+ num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
`input_ids` above)
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1792,12 +1795,12 @@ class MegatronBertForQuestionAnswering(MegatronBertPreTrainedModel):
r"""
start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the start of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the end of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
diff --git a/src/transformers/models/mluke/tokenization_mluke.py b/src/transformers/models/mluke/tokenization_mluke.py
index 5123e0ef02..679ad289ec 100644
--- a/src/transformers/models/mluke/tokenization_mluke.py
+++ b/src/transformers/models/mluke/tokenization_mluke.py
@@ -84,16 +84,15 @@ ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
return_overflowing_tokens (`bool`, *optional*, defaults to `False`):
Whether or not to return overflowing token sequences. If a pair of sequences of input ids (or a batch
- of pairs) is provided with `truncation_strategy = longest_first` or `True`, an error is
- raised instead of returning overflowing tokens.
+ of pairs) is provided with `truncation_strategy = longest_first` or `True`, an error is raised instead
+ of returning overflowing tokens.
return_special_tokens_mask (`bool`, *optional*, defaults to `False`):
Whether or not to return special tokens mask information.
return_offsets_mapping (`bool`, *optional*, defaults to `False`):
Whether or not to return `(char_start, char_end)` for each token.
- This is only available on fast tokenizers inheriting from
- [`PreTrainedTokenizerFast`], if using Python's tokenizer, this method will raise
- `NotImplementedError`.
+ This is only available on fast tokenizers inheriting from [`PreTrainedTokenizerFast`], if using
+ Python's tokenizer, this method will raise `NotImplementedError`.
return_length (`bool`, *optional*, defaults to `False`):
Whether or not to return the lengths of the encoded inputs.
verbose (`bool`, *optional*, defaults to `True`):
@@ -107,8 +106,8 @@ ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING = r"""
[What are input IDs?](../glossary#input-ids)
- - **token_type_ids** -- List of token type ids to be fed to a model (when `return_token_type_ids=True`
- or if *"token_type_ids"* is in `self.model_input_names`).
+ - **token_type_ids** -- List of token type ids to be fed to a model (when `return_token_type_ids=True` or
+ if *"token_type_ids"* is in `self.model_input_names`).
[What are token type IDs?](../glossary#token-type-ids)
@@ -129,8 +128,7 @@ ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING = r"""
[What are token type IDs?](../glossary#token-type-ids)
- **entity_attention_mask** -- List of indices specifying which entities should be attended to by the model
- (when `return_attention_mask=True` or if *"entity_attention_mask"* is in
- `self.model_input_names`).
+ (when `return_attention_mask=True` or if *"entity_attention_mask"* is in `self.model_input_names`).
[What are attention masks?](../glossary#attention-mask)
@@ -154,8 +152,8 @@ class MLukeTokenizer(PreTrainedTokenizer):
Adapted from [`XLMRobertaTokenizer`] and [`LukeTokenizer`]. Based on
[SentencePiece](https://github.com/google/sentencepiece).
- This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods.
- Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
+ this superclass for more information regarding those methods.
Args:
vocab_file (`str`):
@@ -177,8 +175,8 @@ class MLukeTokenizer(PreTrainedTokenizer):
- When building a sequence using special tokens, this is not the token that is used for the end of
- sequence. The token used is the `sep_token`.
+ When building a sequence using special tokens, this is not the token that is used for the end of sequence.
+ The token used is the `sep_token`.
@@ -199,8 +197,8 @@ class MLukeTokenizer(PreTrainedTokenizer):
modeling. This is the token which the model will try to predict.
task (`str`, *optional*):
Task for which you want to prepare sequences. One of `"entity_classification"`,
- `"entity_pair_classification"`, or `"entity_span_classification"`. If you specify this argument,
- the entity sequence is automatically created based on the given entity span(s).
+ `"entity_pair_classification"`, or `"entity_span_classification"`. If you specify this argument, the entity
+ sequence is automatically created based on the given entity span(s).
max_entity_length (`int`, *optional*, defaults to 32):
The maximum length of `entity_ids`.
max_mention_length (`int`, *optional*, defaults to 30):
@@ -214,7 +212,9 @@ class MLukeTokenizer(PreTrainedTokenizer):
additional_special_tokens (`List[str]`, *optional*, defaults to `["NOTUSED", "NOTUSED"]`):
Additional special tokens used by the tokenizer.
sp_model_kwargs (`dict`, *optional*):
- Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things, to set:
+ Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for
+ SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things,
+ to set:
- `enable_sampling`: Enable subword regularization.
- `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout.
@@ -405,9 +405,9 @@ class MLukeTokenizer(PreTrainedTokenizer):
entity_spans (`List[Tuple[int, int]]`, `List[List[Tuple[int, int]]]`, *optional*):
The sequence or batch of sequences of entity spans to be encoded. Each sequence consists of tuples each
with two integers denoting character-based start and end positions of entities. If you specify
- `"entity_classification"` or `"entity_pair_classification"` as the `task` argument in the
- constructor, the length of each sequence must be 1 or 2, respectively. If you specify `entities`, the
- length of each sequence must be equal to the length of each sequence of `entities`.
+ `"entity_classification"` or `"entity_pair_classification"` as the `task` argument in the constructor,
+ the length of each sequence must be 1 or 2, respectively. If you specify `entities`, the length of each
+ sequence must be equal to the length of each sequence of `entities`.
entity_spans_pair (`List[Tuple[int, int]]`, `List[List[Tuple[int, int]]]`, *optional*):
The sequence or batch of sequences of entity spans to be encoded. Each sequence consists of tuples each
with two integers denoting character-based start and end positions of entities. If you specify the
@@ -416,15 +416,15 @@ class MLukeTokenizer(PreTrainedTokenizer):
entities (`List[str]`, `List[List[str]]`, *optional*):
The sequence or batch of sequences of entities to be encoded. Each sequence consists of strings
representing entities, i.e., special entities (e.g., [MASK]) or entity titles of Wikipedia (e.g., Los
- Angeles). This argument is ignored if you specify the `task` argument in the constructor. The length
- of each sequence must be equal to the length of each sequence of `entity_spans`. If you specify
+ Angeles). This argument is ignored if you specify the `task` argument in the constructor. The length of
+ each sequence must be equal to the length of each sequence of `entity_spans`. If you specify
`entity_spans` without specifying this argument, the entity sequence or the batch of entity sequences
is automatically constructed by filling it with the [MASK] entity.
entities_pair (`List[str]`, `List[List[str]]`, *optional*):
The sequence or batch of sequences of entities to be encoded. Each sequence consists of strings
representing entities, i.e., special entities (e.g., [MASK]) or entity titles of Wikipedia (e.g., Los
- Angeles). This argument is ignored if you specify the `task` argument in the constructor. The length
- of each sequence must be equal to the length of each sequence of `entity_spans_pair`. If you specify
+ Angeles). This argument is ignored if you specify the `task` argument in the constructor. The length of
+ each sequence must be equal to the length of each sequence of `entity_spans_pair`. If you specify
`entity_spans_pair` without specifying this argument, the entity sequence or the batch of entity
sequences is automatically constructed by filling it with the [MASK] entity.
max_entity_length (`int`, *optional*):
@@ -1210,27 +1210,28 @@ class MLukeTokenizer(PreTrainedTokenizer):
"""
Pad a single encoded input or a batch of encoded inputs up to predefined length or to the max sequence length
in the batch. Padding side (left/right) padding token ids are defined at the tokenizer level (with
- `self.padding_side`, `self.pad_token_id` and `self.pad_token_type_id`) .. note:: If the
- `encoded_inputs` passed are dictionary of numpy arrays, PyTorch tensors or TensorFlow tensors, the result
- will use the same type unless you provide a different tensor type with `return_tensors`. In the case of
- PyTorch tensors, you will lose the specific device of your tensors however.
+ `self.padding_side`, `self.pad_token_id` and `self.pad_token_type_id`) .. note:: If the `encoded_inputs` passed
+ are dictionary of numpy arrays, PyTorch tensors or TensorFlow tensors, the result will use the same type unless
+ you provide a different tensor type with `return_tensors`. In the case of PyTorch tensors, you will lose the
+ specific device of your tensors however.
Args:
encoded_inputs ([`BatchEncoding`], list of [`BatchEncoding`], `Dict[str, List[int]]`, `Dict[str, List[List[int]]` or `List[Dict[str, List[int]]]`):
- Tokenized inputs. Can represent one input ([`BatchEncoding`] or `Dict[str, List[int]]`) or a batch of tokenized inputs (list of [`BatchEncoding`], *Dict[str,
- List[List[int]]]* or *List[Dict[str, List[int]]]*) so you can use this method during preprocessing as
- well as in a PyTorch Dataloader collate function. Instead of `List[int]` you can have tensors
- (numpy arrays, PyTorch tensors or TensorFlow tensors), see the note above for the return type.
+ Tokenized inputs. Can represent one input ([`BatchEncoding`] or `Dict[str, List[int]]`) or a batch of
+ tokenized inputs (list of [`BatchEncoding`], *Dict[str, List[List[int]]]* or *List[Dict[str,
+ List[int]]]*) so you can use this method during preprocessing as well as in a PyTorch Dataloader
+ collate function. Instead of `List[int]` you can have tensors (numpy arrays, PyTorch tensors or
+ TensorFlow tensors), see the note above for the return type.
padding (`bool`, `str` or [`~file_utils.PaddingStrategy`], *optional*, defaults to `True`):
Select a strategy to pad the returned sequences (according to the model's padding side and padding
index) among:
- - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a
- single sequence if provided).
- - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the
- maximum acceptable input length for the model if that argument is not provided.
- - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of
- different lengths).
+ - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
+ sequence if provided).
+ - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
+ acceptable input length for the model if that argument is not provided.
+ - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different
+ lengths).
max_length (`int`, *optional*):
Maximum length of the returned list and optionally padding length (see above).
max_entity_length (`int`, *optional*):
@@ -1240,8 +1241,8 @@ class MLukeTokenizer(PreTrainedTokenizer):
the use of Tensor Cores on NVIDIA hardware with compute capability >= 7.5 (Volta).
return_attention_mask (`bool`, *optional*):
Whether to return the attention mask. If left to the default, will return the attention mask according
- to the specific tokenizer's default, defined by the `return_outputs` attribute. [What are
- attention masks?](../glossary#attention-mask)
+ to the specific tokenizer's default, defined by the `return_outputs` attribute. [What are attention
+ masks?](../glossary#attention-mask)
return_tensors (`str` or [`~file_utils.TensorType`], *optional*):
If set, will return tensors instead of list of python integers. Acceptable values are:
@@ -1364,7 +1365,8 @@ class MLukeTokenizer(PreTrainedTokenizer):
Args:
- encoded_inputs: Dictionary of tokenized inputs (`List[int]`) or batch of tokenized inputs (`List[List[int]]`).
+ encoded_inputs:
+ Dictionary of tokenized inputs (`List[int]`) or batch of tokenized inputs (`List[List[int]]`).
max_length: maximum length of the returned list and optionally padding length (see below).
Will truncate by taking into account the special tokens.
max_entity_length: The maximum length of the entity sequence.
@@ -1382,7 +1384,8 @@ class MLukeTokenizer(PreTrainedTokenizer):
pad_to_multiple_of: (optional) Integer if set will pad the sequence to a multiple of the provided value.
This is especially useful to enable the use of Tensor Core on NVIDIA hardware with compute capability
>= 7.5 (Volta).
- return_attention_mask: (optional) Set to False to avoid returning attention mask (default: set to model specifics)
+ return_attention_mask:
+ (optional) Set to False to avoid returning attention mask (default: set to model specifics)
"""
entities_provided = bool("entity_ids" in encoded_inputs)
diff --git a/src/transformers/models/mmbt/configuration_mmbt.py b/src/transformers/models/mmbt/configuration_mmbt.py
index 1137917c34..aa453db592 100644
--- a/src/transformers/models/mmbt/configuration_mmbt.py
+++ b/src/transformers/models/mmbt/configuration_mmbt.py
@@ -13,7 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" MMBT configuration """
+""" MMBT configuration"""
from ...utils import logging
@@ -23,8 +23,8 @@ logger = logging.get_logger(__name__)
class MMBTConfig(object):
"""
- This is the configuration class to store the configuration of a [`MMBTModel`]. It is used to
- instantiate a MMBT model according to the specified arguments, defining the model architecture.
+ This is the configuration class to store the configuration of a [`MMBTModel`]. It is used to instantiate a MMBT
+ model according to the specified arguments, defining the model architecture.
Args:
config ([`PreTrainedConfig`]):
diff --git a/src/transformers/models/mmbt/modeling_mmbt.py b/src/transformers/models/mmbt/modeling_mmbt.py
index d30d31fba6..7b8b3c5ce6 100644
--- a/src/transformers/models/mmbt/modeling_mmbt.py
+++ b/src/transformers/models/mmbt/modeling_mmbt.py
@@ -13,7 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-"""PyTorch MMBT model. """
+"""PyTorch MMBT model."""
import torch
@@ -77,17 +77,18 @@ class ModalEmbeddings(nn.Module):
MMBT_START_DOCSTRING = r"""
- MMBT model was proposed in [Supervised Multimodal Bitransformers for Classifying Images and Text](https://github.com/facebookresearch/mmbt) by Douwe Kiela, Suvrat Bhooshan, Hamed Firooz, Davide Testuggine.
+ MMBT model was proposed in [Supervised Multimodal Bitransformers for Classifying Images and
+ Text](https://github.com/facebookresearch/mmbt) by Douwe Kiela, Suvrat Bhooshan, Hamed Firooz, Davide Testuggine.
It's a supervised multimodal bitransformer model that fuses information from text and other image encoders, and
obtain state-of-the-art performance on various multimodal classification benchmark tasks.
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
Parameters:
config ([`MMBTConfig`]): Model configuration class with all the parameters of the model.
@@ -106,9 +107,8 @@ MMBT_INPUTS_DOCSTRING = r"""
Encoder, the shape would be (batch_size, channels, height, width)
input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
Indices of input sequence tokens in the vocabulary. It does not expect [CLS] token to be added as it's
- appended to the end of other modality embeddings. Indices can be obtained using
- [`BertTokenizer`]. See [`PreTrainedTokenizer.encode`] and
- [`PreTrainedTokenizer.__call__`] for details.
+ appended to the end of other modality embeddings. Indices can be obtained using [`BertTokenizer`]. See
+ [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
modal_start_tokens (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
@@ -124,7 +124,8 @@ MMBT_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (*optional*) `torch.LongTensor` of shape `(batch_size, sequence_length)`:
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
@@ -134,7 +135,8 @@ MMBT_INPUTS_DOCSTRING = r"""
Segment token indices to indicate different portions of the non-text modality. The embeddings from these
tokens will be summed with the respective token embeddings for the non-text modality.
position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
modal_position_ids (`torch.LongTensor` of shape `(batch_size, modal_sequence_length)`, *optional*):
@@ -149,9 +151,9 @@ MMBT_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, embedding_dim)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if
the model is configured as a decoder.
@@ -312,12 +314,15 @@ class MMBTModel(nn.Module, ModuleUtilsMixin):
class MMBTForClassification(nn.Module):
r"""
**labels**: (*optional*) `torch.LongTensor` of shape `(batch_size,)`:
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
Returns: *Tuple* comprising various elements depending on the configuration (config) and inputs: **loss**:
(*optional*, returned when `labels` is provided) `torch.FloatTensor` of shape `(1,)`: Classification (or
- regression if config.num_labels==1) loss. **logits**: `torch.FloatTensor` of shape `(batch_size, config.num_labels)` Classification (or regression if config.num_labels==1) scores (before SoftMax).
+ regression if config.num_labels==1) loss. **logits**:
+ `torch.FloatTensor` of shape `(batch_size, config.num_labels)` Classification (or regression if
+ config.num_labels==1) scores (before SoftMax).
**hidden_states**: (*optional*, returned when `output_hidden_states=True`) list of `torch.FloatTensor` (one for
the output of each layer + the output of the embeddings) of shape `(batch_size, sequence_length, hidden_size)`:
Hidden-states of the model at the output of each layer plus the initial embedding outputs. **attentions**:
diff --git a/src/transformers/models/mobilebert/configuration_mobilebert.py b/src/transformers/models/mobilebert/configuration_mobilebert.py
index a738fc54c7..45be50aaba 100644
--- a/src/transformers/models/mobilebert/configuration_mobilebert.py
+++ b/src/transformers/models/mobilebert/configuration_mobilebert.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" MobileBERT model configuration """
+""" MobileBERT model configuration"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging
@@ -27,19 +27,17 @@ MOBILEBERT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class MobileBertConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`MobileBertModel`] or a
- [`TFMobileBertModel`]. It is used to instantiate a MobileBERT model according to the specified
- arguments, defining the model architecture.
+ This is the configuration class to store the configuration of a [`MobileBertModel`] or a [`TFMobileBertModel`]. It
+ is used to instantiate a MobileBERT model according to the specified arguments, defining the model architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
vocab_size (`int`, *optional*, defaults to 30522):
Vocabulary size of the MobileBERT model. Defines the number of different tokens that can be represented by
- the `inputs_ids` passed when calling [`MobileBertModel`] or
- [`TFMobileBertModel`].
+ the `inputs_ids` passed when calling [`MobileBertModel`] or [`TFMobileBertModel`].
hidden_size (`int`, *optional*, defaults to 512):
Dimensionality of the encoder layers and the pooler layer.
num_hidden_layers (`int`, *optional*, defaults to 24):
@@ -49,8 +47,8 @@ class MobileBertConfig(PretrainedConfig):
intermediate_size (`int`, *optional*, defaults to 512):
Dimensionality of the "intermediate" (often named feed-forward) layer in the Transformer encoder.
hidden_act (`str` or `function`, *optional*, defaults to `"relu"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"silu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"silu"` and `"gelu_new"` are supported.
hidden_dropout_prob (`float`, *optional*, defaults to 0.0):
The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
attention_probs_dropout_prob (`float`, *optional*, defaults to 0.1):
@@ -59,8 +57,8 @@ class MobileBertConfig(PretrainedConfig):
The maximum sequence length that this model might ever be used with. Typically set this to something large
just in case (e.g., 512 or 1024 or 2048).
type_vocab_size (`int`, *optional*, defaults to 2):
- The vocabulary size of the `token_type_ids` passed when calling [`MobileBertModel`]
- or [`TFMobileBertModel`].
+ The vocabulary size of the `token_type_ids` passed when calling [`MobileBertModel`] or
+ [`TFMobileBertModel`].
initializer_range (`float`, *optional*, defaults to 0.02):
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
layer_norm_eps (`float`, *optional*, defaults to 1e-12):
diff --git a/src/transformers/models/mobilebert/modeling_mobilebert.py b/src/transformers/models/mobilebert/modeling_mobilebert.py
index 38738daeb9..57bc42edbe 100644
--- a/src/transformers/models/mobilebert/modeling_mobilebert.py
+++ b/src/transformers/models/mobilebert/modeling_mobilebert.py
@@ -695,12 +695,13 @@ class MobileBertForPreTrainingOutput(ModelOutput):
Prediction scores of the next sequence prediction (classification) head (scores of True/False continuation
before SoftMax).
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -715,19 +716,18 @@ class MobileBertForPreTrainingOutput(ModelOutput):
MOBILEBERT_START_DOCSTRING = r"""
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
Parameters:
config ([`MobileBertConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
MOBILEBERT_INPUTS_DOCSTRING = r"""
@@ -735,9 +735,8 @@ MOBILEBERT_INPUTS_DOCSTRING = r"""
input_ids (`torch.LongTensor` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`BertTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`BertTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
@@ -748,14 +747,16 @@ MOBILEBERT_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
@@ -765,9 +766,9 @@ MOBILEBERT_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -940,8 +941,9 @@ class MobileBertForPreTraining(MobileBertPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
next_sentence_label (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair
(see `input_ids` docstring) Indices should be in `[0, 1]`:
@@ -1002,7 +1004,7 @@ class MobileBertForPreTraining(MobileBertPreTrainedModel):
)
-@add_start_docstrings("""MobileBert Model with a `language modeling` head on top. """, MOBILEBERT_START_DOCSTRING)
+@add_start_docstrings("""MobileBert Model with a `language modeling` head on top.""", MOBILEBERT_START_DOCSTRING)
class MobileBertForMaskedLM(MobileBertPreTrainedModel):
_keys_to_ignore_on_load_unexpected = [r"pooler"]
@@ -1051,8 +1053,9 @@ class MobileBertForMaskedLM(MobileBertPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1099,7 +1102,7 @@ class MobileBertOnlyNSPHead(nn.Module):
@add_start_docstrings(
- """MobileBert Model with a `next sentence prediction (classification)` head on top. """,
+ """MobileBert Model with a `next sentence prediction (classification)` head on top.""",
MOBILEBERT_START_DOCSTRING,
)
class MobileBertForNextSentencePrediction(MobileBertPreTrainedModel):
diff --git a/src/transformers/models/mobilebert/modeling_tf_mobilebert.py b/src/transformers/models/mobilebert/modeling_tf_mobilebert.py
index db7ad2ae09..e4a8b73029 100644
--- a/src/transformers/models/mobilebert/modeling_tf_mobilebert.py
+++ b/src/transformers/models/mobilebert/modeling_tf_mobilebert.py
@@ -13,7 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" TF 2.0 MobileBERT model. """
+""" TF 2.0 MobileBERT model."""
import warnings
from dataclasses import dataclass
@@ -808,12 +808,13 @@ class TFMobileBertForPreTrainingOutput(ModelOutput):
Prediction scores of the next sequence prediction (classification) head (scores of True/False continuation
before SoftMax).
hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -828,13 +829,13 @@ class TFMobileBertForPreTrainingOutput(ModelOutput):
MOBILEBERT_START_DOCSTRING = r"""
- This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading or saving, resizing the input
- embeddings, pruning heads etc.)
+ This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use
- it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage
- and behavior.
+ This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it
+ as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and
+ behavior.
@@ -843,11 +844,11 @@ MOBILEBERT_START_DOCSTRING = r"""
- having all inputs as keyword arguments (like PyTorch models), or
- having all inputs as a list, tuple or dict in the first positional arguments.
- This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all
- the tensors in the first argument of the model call function: `model(inputs)`.
+ This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all the
+ tensors in the first argument of the model call function: `model(inputs)`.
- If you choose this second option, there are three possibilities you can use to gather all the input Tensors in
- the first positional argument :
+ If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the
+ first positional argument :
- a single Tensor with `input_ids` only and nothing else: `model(inputs_ids)`
- a list of varying length with one or several input Tensors IN THE ORDER given in the docstring:
@@ -860,8 +861,7 @@ MOBILEBERT_START_DOCSTRING = r"""
Parameters:
config ([`MobileBertConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
MOBILEBERT_INPUTS_DOCSTRING = r"""
@@ -869,9 +869,8 @@ MOBILEBERT_INPUTS_DOCSTRING = r"""
input_ids (`Numpy array` or `tf.Tensor` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`MobileBertTokenizer`]. See
- [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for
- details.
+ Indices can be obtained using [`MobileBertTokenizer`]. See [`PreTrainedTokenizer.__call__`] and
+ [`PreTrainedTokenizer.encode`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`Numpy array` or `tf.Tensor` of shape `({0})`, *optional*):
@@ -882,14 +881,16 @@ MOBILEBERT_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`Numpy array` or `tf.Tensor` of shape `({0})`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`Numpy array` or `tf.Tensor` of shape `({0})`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
head_mask (`Numpy array` or `tf.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
@@ -899,9 +900,9 @@ MOBILEBERT_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
inputs_embeds (`tf.Tensor` of shape `({0}, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the
@@ -911,8 +912,8 @@ MOBILEBERT_INPUTS_DOCSTRING = r"""
more detail. This argument can be used only in eager mode, in graph mode the value in the config will be
used instead.
return_dict (`bool`, *optional*):
- Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This
- argument can be used in eager mode, in graph mode the value will always be set to True.
+ Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This argument can be used
+ in eager mode, in graph mode the value will always be set to True.
training (`bool`, *optional*, defaults to `False`):
Whether or not to use the model in training mode (some modules like dropout modules have different
behaviors between training and evaluation).
@@ -1097,7 +1098,7 @@ class TFMobileBertForPreTraining(TFMobileBertPreTrainedModel):
)
-@add_start_docstrings("""MobileBert Model with a `language modeling` head on top. """, MOBILEBERT_START_DOCSTRING)
+@add_start_docstrings("""MobileBert Model with a `language modeling` head on top.""", MOBILEBERT_START_DOCSTRING)
class TFMobileBertForMaskedLM(TFMobileBertPreTrainedModel, TFMaskedLanguageModelingLoss):
# names with a '.' represents the authorized unexpected/missing layers when a TF model is loaded from a PT model
_keys_to_ignore_on_load_unexpected = [
@@ -1143,8 +1144,9 @@ class TFMobileBertForMaskedLM(TFMobileBertPreTrainedModel, TFMaskedLanguageModel
):
r"""
labels (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+ loss is only computed for the tokens with labels
"""
inputs = input_processing(
func=self.call,
@@ -1209,7 +1211,7 @@ class TFMobileBertOnlyNSPHead(tf.keras.layers.Layer):
@add_start_docstrings(
- """MobileBert Model with a `next sentence prediction (classification)` head on top. """,
+ """MobileBert Model with a `next sentence prediction (classification)` head on top.""",
MOBILEBERT_START_DOCSTRING,
)
class TFMobileBertForNextSentencePrediction(TFMobileBertPreTrainedModel, TFNextSentencePredictionLoss):
@@ -1367,8 +1369,9 @@ class TFMobileBertForSequenceClassification(TFMobileBertPreTrainedModel, TFSeque
):
r"""
labels (`tf.Tensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
inputs = input_processing(
func=self.call,
@@ -1476,12 +1479,12 @@ class TFMobileBertForQuestionAnswering(TFMobileBertPreTrainedModel, TFQuestionAn
r"""
start_positions (`tf.Tensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the start of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
end_positions (`tf.Tensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the end of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
"""
inputs = input_processing(
func=self.call,
@@ -1609,8 +1612,8 @@ class TFMobileBertForMultipleChoice(TFMobileBertPreTrainedModel, TFMultipleChoic
):
r"""
labels (`tf.Tensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices]` where `num_choices` is the size of the second dimension of the input tensors. (See
- `input_ids` above)
+ Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices]`
+ where `num_choices` is the size of the second dimension of the input tensors. (See `input_ids` above)
"""
inputs = input_processing(
func=self.call,
diff --git a/src/transformers/models/mobilebert/tokenization_mobilebert.py b/src/transformers/models/mobilebert/tokenization_mobilebert.py
index ef9828c407..509b752ed6 100644
--- a/src/transformers/models/mobilebert/tokenization_mobilebert.py
+++ b/src/transformers/models/mobilebert/tokenization_mobilebert.py
@@ -37,11 +37,10 @@ class MobileBertTokenizer(BertTokenizer):
r"""
Construct a MobileBERT tokenizer.
- [`MobileBertTokenizer`] is identical to [`BertTokenizer`] and runs end-to-end
- tokenization: punctuation splitting and wordpiece.
+ [`MobileBertTokenizer`] is identical to [`BertTokenizer`] and runs end-to-end tokenization: punctuation splitting
+ and wordpiece.
- Refer to superclass [`BertTokenizer`] for usage examples and documentation concerning
- parameters.
+ Refer to superclass [`BertTokenizer`] for usage examples and documentation concerning parameters.
"""
vocab_files_names = VOCAB_FILES_NAMES
diff --git a/src/transformers/models/mobilebert/tokenization_mobilebert_fast.py b/src/transformers/models/mobilebert/tokenization_mobilebert_fast.py
index 28eced0356..819d16c5c1 100644
--- a/src/transformers/models/mobilebert/tokenization_mobilebert_fast.py
+++ b/src/transformers/models/mobilebert/tokenization_mobilebert_fast.py
@@ -41,11 +41,10 @@ class MobileBertTokenizerFast(BertTokenizerFast):
r"""
Construct a "fast" MobileBERT tokenizer (backed by HuggingFace's *tokenizers* library).
- [`MobileBertTokenizerFast`] is identical to [`BertTokenizerFast`] and runs
- end-to-end tokenization: punctuation splitting and wordpiece.
+ [`MobileBertTokenizerFast`] is identical to [`BertTokenizerFast`] and runs end-to-end tokenization: punctuation
+ splitting and wordpiece.
- Refer to superclass [`BertTokenizerFast`] for usage examples and documentation concerning
- parameters.
+ Refer to superclass [`BertTokenizerFast`] for usage examples and documentation concerning parameters.
"""
vocab_files_names = VOCAB_FILES_NAMES
diff --git a/src/transformers/models/mpnet/configuration_mpnet.py b/src/transformers/models/mpnet/configuration_mpnet.py
index a6fc99486b..d1b0bb07db 100644
--- a/src/transformers/models/mpnet/configuration_mpnet.py
+++ b/src/transformers/models/mpnet/configuration_mpnet.py
@@ -13,7 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" MPNet model configuration """
+""" MPNet model configuration"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging
@@ -28,19 +28,18 @@ MPNET_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class MPNetConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`MPNetModel`] or a
- [`TFMPNetModel`]. It is used to instantiate a MPNet model according to the specified arguments,
- defining the model architecture. Instantiating a configuration with the defaults will yield a similar configuration
- to that of the MPNet [mpnet-base](https://huggingface.co/mpnet-base) architecture.
+ This is the configuration class to store the configuration of a [`MPNetModel`] or a [`TFMPNetModel`]. It is used to
+ instantiate a MPNet model according to the specified arguments, defining the model architecture. Instantiating a
+ configuration with the defaults will yield a similar configuration to that of the MPNet
+ [mpnet-base](https://huggingface.co/mpnet-base) architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
vocab_size (`int`, *optional*, defaults to 30527):
Vocabulary size of the MPNet model. Defines the number of different tokens that can be represented by the
- `inputs_ids` passed when calling [`MPNetModel`] or
- [`TFMPNetModel`].
+ `inputs_ids` passed when calling [`MPNetModel`] or [`TFMPNetModel`].
hidden_size (`int`, *optional*, defaults to 768):
Dimensionality of the encoder layers and the pooler layer.
num_hidden_layers (`int`, *optional*, defaults to 12):
@@ -50,8 +49,8 @@ class MPNetConfig(PretrainedConfig):
intermediate_size (`int`, *optional*, defaults to 3072):
Dimensionality of the "intermediate" (often named feed-forward) layer in the Transformer encoder.
hidden_act (`str` or `Callable`, *optional*, defaults to `"gelu"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"silu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"silu"` and `"gelu_new"` are supported.
hidden_dropout_prob (`float`, *optional*, defaults to 0.1):
The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
attention_probs_dropout_prob (`float`, *optional*, defaults to 0.1):
diff --git a/src/transformers/models/mpnet/modeling_mpnet.py b/src/transformers/models/mpnet/modeling_mpnet.py
index 59ce04003e..8a2698969a 100644
--- a/src/transformers/models/mpnet/modeling_mpnet.py
+++ b/src/transformers/models/mpnet/modeling_mpnet.py
@@ -13,7 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-"""PyTorch MPNet model. """
+"""PyTorch MPNet model."""
import math
@@ -419,19 +419,18 @@ class MPNetPooler(nn.Module):
MPNET_START_DOCSTRING = r"""
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
Parameters:
config ([`MPNetConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
MPNET_INPUTS_DOCSTRING = r"""
@@ -439,9 +438,8 @@ MPNET_INPUTS_DOCSTRING = r"""
input_ids (`torch.LongTensor` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`MPNetTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`MPNetTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -452,7 +450,8 @@ MPNET_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
@@ -462,9 +461,9 @@ MPNET_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert *input_ids* indices into associated vectors
- than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -613,8 +612,9 @@ class MPNetForMaskedLM(MPNetPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -715,8 +715,9 @@ class MPNetForSequenceClassification(MPNetPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -809,7 +810,8 @@ class MPNetForMultipleChoice(MPNetPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
+ Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
+ num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
`input_ids` above)
"""
@@ -1011,12 +1013,12 @@ class MPNetForQuestionAnswering(MPNetPreTrainedModel):
r"""
start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the start of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the end of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
diff --git a/src/transformers/models/mpnet/modeling_tf_mpnet.py b/src/transformers/models/mpnet/modeling_tf_mpnet.py
index c2322d13d5..4778249f30 100644
--- a/src/transformers/models/mpnet/modeling_tf_mpnet.py
+++ b/src/transformers/models/mpnet/modeling_tf_mpnet.py
@@ -13,7 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" TF 2.0 MPNet model. """
+""" TF 2.0 MPNet model."""
import math
@@ -588,13 +588,13 @@ class TFMPNetMainLayer(tf.keras.layers.Layer):
MPNET_START_DOCSTRING = r"""
- This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading or saving, resizing the input
- embeddings, pruning heads etc.)
+ This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use
- it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage
- and behavior.
+ This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it
+ as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and
+ behavior.
@@ -603,11 +603,11 @@ MPNET_START_DOCSTRING = r"""
- having all inputs as keyword arguments (like PyTorch models), or
- having all inputs as a list, tuple or dict in the first positional arguments.
- This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all
- the tensors in the first argument of the model call function: `model(inputs)`.
+ This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all the
+ tensors in the first argument of the model call function: `model(inputs)`.
- If you choose this second option, there are three possibilities you can use to gather all the input Tensor in
- the first positional argument :
+ If you choose this second option, there are three possibilities you can use to gather all the input Tensor in the
+ first positional argument :
- a single Tensor with `input_ids` only and nothing else: `model(inputs_ids)`
- a list of varying length with one or several input Tensors IN THE ORDER given in the docstring:
@@ -620,8 +620,7 @@ MPNET_START_DOCSTRING = r"""
Args:
config ([`MPNetConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
MPNET_INPUTS_DOCSTRING = r"""
@@ -629,9 +628,8 @@ MPNET_INPUTS_DOCSTRING = r"""
input_ids (`Numpy array` or `tf.Tensor` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`MPNetTokenizer`]. See
- [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for
- details.
+ Indices can be obtained using [`MPNetTokenizer`]. See [`PreTrainedTokenizer.__call__`] and
+ [`PreTrainedTokenizer.encode`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`Numpy array` or `tf.Tensor` of shape `({0})`, *optional*):
@@ -642,7 +640,8 @@ MPNET_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
position_ids (`Numpy array` or `tf.Tensor` of shape `({0})`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
head_mask (`Numpy array` or `tf.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
@@ -652,9 +651,9 @@ MPNET_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
inputs_embeds (`tf.Tensor` of shape `({0}, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the
@@ -664,8 +663,8 @@ MPNET_INPUTS_DOCSTRING = r"""
more detail. This argument can be used only in eager mode, in graph mode the value in the config will be
used instead.
return_dict (`bool`, *optional*):
- Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This
- argument can be used in eager mode, in graph mode the value will always be set to True.
+ Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This argument can be used
+ in eager mode, in graph mode the value will always be set to True.
training (`bool`, *optional*, defaults to `False`):
Whether or not to use the model in training mode (some modules like dropout modules have different
behaviors between training and evaluation).
@@ -792,7 +791,7 @@ class TFMPNetLMHead(tf.keras.layers.Layer):
return hidden_states
-@add_start_docstrings("""MPNet Model with a `language modeling` head on top. """, MPNET_START_DOCSTRING)
+@add_start_docstrings("""MPNet Model with a `language modeling` head on top.""", MPNET_START_DOCSTRING)
class TFMPNetForMaskedLM(TFMPNetPreTrainedModel, TFMaskedLanguageModelingLoss):
_keys_to_ignore_on_load_missing = [r"pooler"]
@@ -833,8 +832,9 @@ class TFMPNetForMaskedLM(TFMPNetPreTrainedModel, TFMaskedLanguageModelingLoss):
):
r"""
labels (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
"""
inputs = input_processing(
@@ -953,8 +953,9 @@ class TFMPNetForSequenceClassification(TFMPNetPreTrainedModel, TFSequenceClassif
):
r"""
labels (`tf.Tensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
inputs = input_processing(
@@ -1058,8 +1059,8 @@ class TFMPNetForMultipleChoice(TFMPNetPreTrainedModel, TFMultipleChoiceLoss):
):
r"""
labels (`tf.Tensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices]` where `num_choices` is the size of the second dimension of the input tensors. (See
- `input_ids` above)
+ Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices]`
+ where `num_choices` is the size of the second dimension of the input tensors. (See `input_ids` above)
"""
inputs = input_processing(
func=self.call,
@@ -1289,12 +1290,12 @@ class TFMPNetForQuestionAnswering(TFMPNetPreTrainedModel, TFQuestionAnsweringLos
r"""
start_positions (`tf.Tensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the start of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
end_positions (`tf.Tensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the end of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
"""
inputs = input_processing(
diff --git a/src/transformers/models/mpnet/tokenization_mpnet.py b/src/transformers/models/mpnet/tokenization_mpnet.py
index 1de9746a21..e82d12c656 100644
--- a/src/transformers/models/mpnet/tokenization_mpnet.py
+++ b/src/transformers/models/mpnet/tokenization_mpnet.py
@@ -66,8 +66,8 @@ def whitespace_tokenize(text):
class MPNetTokenizer(PreTrainedTokenizer):
"""
- This tokenizer inherits from [`BertTokenizer`] which contains most of the methods. Users should
- refer to the superclass for more information regarding methods.
+ This tokenizer inherits from [`BertTokenizer`] which contains most of the methods. Users should refer to the
+ superclass for more information regarding methods.
Args:
vocab_file (`str`):
@@ -94,8 +94,8 @@ class MPNetTokenizer(PreTrainedTokenizer):
- When building a sequence using special tokens, this is not the token that is used for the end of
- sequence. The token used is the `sep_token`.
+ When building a sequence using special tokens, this is not the token that is used for the end of sequence.
+ The token used is the `sep_token`.
@@ -117,7 +117,8 @@ class MPNetTokenizer(PreTrainedTokenizer):
tokenize_chinese_chars (`bool`, *optional*, defaults to `True`):
Whether or not to tokenize Chinese characters.
- This should likely be deactivated for Japanese (see this [issue](https://github.com/huggingface/transformers/issues/328)).
+ This should likely be deactivated for Japanese (see this
+ [issue](https://github.com/huggingface/transformers/issues/328)).
strip_accents: (`bool`, *optional*):
Whether or not to strip all accents. If this option is not specified, then it will be determined by the
value for `lowercase` (as in the original BERT).
@@ -337,7 +338,8 @@ class BasicTokenizer(object):
tokenize_chinese_chars (`bool`, *optional*, defaults to `True`):
Whether or not to tokenize Chinese characters.
- This should likely be deactivated for Japanese (see this [issue](https://github.com/huggingface/transformers/issues/328)).
+ This should likely be deactivated for Japanese (see this
+ [issue](https://github.com/huggingface/transformers/issues/328)).
strip_accents: (`bool`, *optional*):
Whether or not to strip all accents. If this option is not specified, then it will be determined by the
value for `lowercase` (as in the original BERT).
diff --git a/src/transformers/models/mpnet/tokenization_mpnet_fast.py b/src/transformers/models/mpnet/tokenization_mpnet_fast.py
index 87b50e144a..c913f85682 100644
--- a/src/transformers/models/mpnet/tokenization_mpnet_fast.py
+++ b/src/transformers/models/mpnet/tokenization_mpnet_fast.py
@@ -52,8 +52,8 @@ class MPNetTokenizerFast(PreTrainedTokenizerFast):
r"""
Construct a "fast" MPNet tokenizer (backed by HuggingFace's *tokenizers* library). Based on WordPiece.
- This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main
- methods. Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should
+ refer to this superclass for more information regarding those methods.
Args:
vocab_file (`str`):
@@ -75,8 +75,8 @@ class MPNetTokenizerFast(PreTrainedTokenizerFast):
- When building a sequence using special tokens, this is not the token that is used for the end of
- sequence. The token used is the `sep_token`.
+ When building a sequence using special tokens, this is not the token that is used for the end of sequence.
+ The token used is the `sep_token`.
@@ -157,8 +157,8 @@ class MPNetTokenizerFast(PreTrainedTokenizerFast):
@property
def mask_token(self) -> str:
"""
- `str`: Mask token, to use when training a model with masked-language modeling. Log an error if used while
- not having been set.
+ `str`: Mask token, to use when training a model with masked-language modeling. Log an error if used while not
+ having been set.
MPNet tokenizer has a special mask token to be usable in the fill-mask pipeline. The mask token will greedily
comprise the space before the **.
diff --git a/src/transformers/models/mt5/configuration_mt5.py b/src/transformers/models/mt5/configuration_mt5.py
index e2275c5443..a5b3262e01 100644
--- a/src/transformers/models/mt5/configuration_mt5.py
+++ b/src/transformers/models/mt5/configuration_mt5.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" mT5 model configuration """
+""" mT5 model configuration"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging
@@ -23,13 +23,13 @@ logger = logging.get_logger(__name__)
class MT5Config(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`MT5Model`] or a
- [`TFMT5Model`]. It is used to instantiate a mT5 model according to the specified arguments,
- defining the model architecture. Instantiating a configuration with the defaults will yield a similar configuration
- to that of the mT5 [google/mt5-small](https://huggingface.co/google/mt5-small) architecture.
+ This is the configuration class to store the configuration of a [`MT5Model`] or a [`TFMT5Model`]. It is used to
+ instantiate a mT5 model according to the specified arguments, defining the model architecture. Instantiating a
+ configuration with the defaults will yield a similar configuration to that of the mT5
+ [google/mt5-small](https://huggingface.co/google/mt5-small) architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Arguments:
vocab_size (`int`, *optional*, defaults to 250112):
@@ -38,14 +38,14 @@ class MT5Config(PretrainedConfig):
d_model (`int`, *optional*, defaults to 512):
Size of the encoder layers and the pooler layer.
d_kv (`int`, *optional*, defaults to 64):
- Size of the key, query, value projections per attention head. `d_kv` has to be equal to `d_model // num_heads`.
+ Size of the key, query, value projections per attention head. `d_kv` has to be equal to `d_model //
+ num_heads`.
d_ff (`int`, *optional*, defaults to 1024):
Size of the intermediate feed forward layer in each `T5Block`.
num_layers (`int`, *optional*, defaults to 8):
Number of hidden layers in the Transformer encoder.
num_decoder_layers (`int`, *optional*):
- Number of hidden layers in the Transformer decoder. Will use the same value as `num_layers` if not
- set.
+ Number of hidden layers in the Transformer decoder. Will use the same value as `num_layers` if not set.
num_heads (`int`, *optional*, defaults to 6):
Number of attention heads for each attention layer in the Transformer encoder.
relative_attention_num_buckets (`int`, *optional*, defaults to 32):
diff --git a/src/transformers/models/mt5/modeling_flax_mt5.py b/src/transformers/models/mt5/modeling_flax_mt5.py
index 43abc1794c..d45ea49645 100644
--- a/src/transformers/models/mt5/modeling_flax_mt5.py
+++ b/src/transformers/models/mt5/modeling_flax_mt5.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" Flax mT5 model. """
+""" Flax mT5 model."""
from ...utils import logging
from ..t5.modeling_flax_t5 import FlaxT5ForConditionalGeneration, FlaxT5Model
@@ -27,8 +27,8 @@ _TOKENIZER_FOR_DOC = "T5Tokenizer"
class FlaxMT5Model(FlaxT5Model):
r"""
- This class overrides [`FlaxT5Model`]. Please check the superclass for the appropriate
- documentation alongside usage examples.
+ This class overrides [`FlaxT5Model`]. Please check the superclass for the appropriate documentation alongside usage
+ examples.
Examples:
@@ -54,8 +54,8 @@ class FlaxMT5Model(FlaxT5Model):
class FlaxMT5ForConditionalGeneration(FlaxT5ForConditionalGeneration):
r"""
- This class overrides [`FlaxT5ForConditionalGeneration`]. Please check the superclass for the
- appropriate documentation alongside usage examples.
+ This class overrides [`FlaxT5ForConditionalGeneration`]. Please check the superclass for the appropriate
+ documentation alongside usage examples.
Examples:
diff --git a/src/transformers/models/mt5/modeling_mt5.py b/src/transformers/models/mt5/modeling_mt5.py
index 87ca5d2b83..128eba98e0 100644
--- a/src/transformers/models/mt5/modeling_mt5.py
+++ b/src/transformers/models/mt5/modeling_mt5.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" PyTorch mT5 model. """
+""" PyTorch mT5 model."""
from ...utils import logging
from ..t5.modeling_t5 import T5EncoderModel, T5ForConditionalGeneration, T5Model
@@ -27,8 +27,8 @@ _TOKENIZER_FOR_DOC = "T5Tokenizer"
class MT5Model(T5Model):
r"""
- This class overrides [`T5Model`]. Please check the superclass for the appropriate documentation
- alongside usage examples.
+ This class overrides [`T5Model`]. Please check the superclass for the appropriate documentation alongside usage
+ examples.
Examples:
@@ -60,8 +60,8 @@ class MT5Model(T5Model):
class MT5ForConditionalGeneration(T5ForConditionalGeneration):
r"""
- This class overrides [`T5ForConditionalGeneration`]. Please check the superclass for the
- appropriate documentation alongside usage examples.
+ This class overrides [`T5ForConditionalGeneration`]. Please check the superclass for the appropriate documentation
+ alongside usage examples.
Examples:
@@ -91,8 +91,8 @@ class MT5ForConditionalGeneration(T5ForConditionalGeneration):
class MT5EncoderModel(T5EncoderModel):
r"""
- This class overrides [`T5EncoderModel`]. Please check the superclass for the appropriate
- documentation alongside usage examples.
+ This class overrides [`T5EncoderModel`]. Please check the superclass for the appropriate documentation alongside
+ usage examples.
Examples:
diff --git a/src/transformers/models/mt5/modeling_tf_mt5.py b/src/transformers/models/mt5/modeling_tf_mt5.py
index 274c30c23a..3b08bb4a02 100644
--- a/src/transformers/models/mt5/modeling_tf_mt5.py
+++ b/src/transformers/models/mt5/modeling_tf_mt5.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" Tensorflow mT5 model. """
+""" Tensorflow mT5 model."""
from ...utils import logging
from ..t5.modeling_tf_t5 import TFT5EncoderModel, TFT5ForConditionalGeneration, TFT5Model
@@ -27,8 +27,8 @@ _TOKENIZER_FOR_DOC = "T5Tokenizer"
class TFMT5Model(TFT5Model):
r"""
- This class overrides [`TFT5Model`]. Please check the superclass for the appropriate
- documentation alongside usage examples.
+ This class overrides [`TFT5Model`]. Please check the superclass for the appropriate documentation alongside usage
+ examples.
Examples:
@@ -51,8 +51,8 @@ class TFMT5Model(TFT5Model):
class TFMT5ForConditionalGeneration(TFT5ForConditionalGeneration):
r"""
- This class overrides [`TFT5ForConditionalGeneration`]. Please check the superclass for the
- appropriate documentation alongside usage examples.
+ This class overrides [`TFT5ForConditionalGeneration`]. Please check the superclass for the appropriate
+ documentation alongside usage examples.
Examples:
@@ -76,8 +76,8 @@ class TFMT5ForConditionalGeneration(TFT5ForConditionalGeneration):
class TFMT5EncoderModel(TFT5EncoderModel):
r"""
- This class overrides [`TFT5EncoderModel`]. Please check the superclass for the appropriate
- documentation alongside usage examples.
+ This class overrides [`TFT5EncoderModel`]. Please check the superclass for the appropriate documentation alongside
+ usage examples.
Examples:
diff --git a/src/transformers/models/openai/configuration_openai.py b/src/transformers/models/openai/configuration_openai.py
index 7f212b6c3f..55ef5fb21d 100644
--- a/src/transformers/models/openai/configuration_openai.py
+++ b/src/transformers/models/openai/configuration_openai.py
@@ -13,7 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" OpenAI GPT configuration """
+""" OpenAI GPT configuration"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging
@@ -26,19 +26,18 @@ OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP = {"openai-gpt": "https://huggingface.c
class OpenAIGPTConfig(PretrainedConfig):
"""
- This is the configuration class to store the configuration of a [`OpenAIGPTModel`] or a
- [`TFOpenAIGPTModel`]. It is used to instantiate a GPT model according to the specified
- arguments, defining the model architecture. Instantiating a configuration with the defaults will yield a similar
- configuration to that of the [GPT](https://huggingface.co/openai-gpt) architecture from OpenAI.
+ This is the configuration class to store the configuration of a [`OpenAIGPTModel`] or a [`TFOpenAIGPTModel`]. It is
+ used to instantiate a GPT model according to the specified arguments, defining the model architecture.
+ Instantiating a configuration with the defaults will yield a similar configuration to that of the
+ [GPT](https://huggingface.co/openai-gpt) architecture from OpenAI.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
vocab_size (`int`, *optional*, defaults to 40478):
Vocabulary size of the GPT-2 model. Defines the number of different tokens that can be represented by the
- `inputs_ids` passed when calling [`OpenAIGPTModel`] or
- [`TFOpenAIGPTModel`].
+ `inputs_ids` passed when calling [`OpenAIGPTModel`] or [`TFOpenAIGPTModel`].
n_positions (`int`, *optional*, defaults to 512):
The maximum sequence length that this model might ever be used with. Typically set this to something large
just in case (e.g., 512 or 1024 or 2048).
@@ -49,8 +48,8 @@ class OpenAIGPTConfig(PretrainedConfig):
n_head (`int`, *optional*, defaults to 12):
Number of attention heads for each attention layer in the Transformer encoder.
afn (`str` or `Callable`, *optional*, defaults to `"gelu"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"silu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"silu"` and `"gelu_new"` are supported.
resid_pdrop (`float`, *optional*, defaults to 0.1):
The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
embd_pdrop (`int`, *optional*, defaults to 0.1):
@@ -64,8 +63,8 @@ class OpenAIGPTConfig(PretrainedConfig):
predict_special_tokens (`bool`, *optional*, defaults to `True`):
Whether or not special tokens should be predicted when the model has a language modeling head.
summary_type (`str`, *optional*, defaults to `"cls_index"`):
- Argument used when doing sequence summary, used in the models
- [`OpenAIGPTDoubleHeadsModel`] and [`OpenAIGPTDoubleHeadsModel`].
+ Argument used when doing sequence summary, used in the models [`OpenAIGPTDoubleHeadsModel`] and
+ [`OpenAIGPTDoubleHeadsModel`].
Has to be one of the following options:
@@ -75,23 +74,23 @@ class OpenAIGPTConfig(PretrainedConfig):
- `"cls_index"`: Supply a Tensor of classification token position (like GPT/GPT-2).
- `"attn"`: Not implemented now, use multi-head attention.
summary_use_proj (`bool`, *optional*, defaults to `True`):
- Argument used when doing sequence summary, used in the models
- [`OpenAIGPTDoubleHeadsModel`] and [`OpenAIGPTDoubleHeadsModel`].
+ Argument used when doing sequence summary, used in the models [`OpenAIGPTDoubleHeadsModel`] and
+ [`OpenAIGPTDoubleHeadsModel`].
Whether or not to add a projection after the vector extraction.
summary_activation (`str`, *optional*):
- Argument used when doing sequence summary, used in the models
- [`OpenAIGPTDoubleHeadsModel`] and [`OpenAIGPTDoubleHeadsModel`].
+ Argument used when doing sequence summary, used in the models [`OpenAIGPTDoubleHeadsModel`] and
+ [`OpenAIGPTDoubleHeadsModel`].
Pass `"tanh"` for a tanh activation to the output, any other value will result in no activation.
summary_proj_to_labels (`bool`, *optional*, defaults to `True`):
- Argument used when doing sequence summary, used in the models
- [`OpenAIGPTDoubleHeadsModel`] and [`OpenAIGPTDoubleHeadsModel`].
+ Argument used when doing sequence summary, used in the models [`OpenAIGPTDoubleHeadsModel`] and
+ [`OpenAIGPTDoubleHeadsModel`].
Whether the projection outputs should have `config.num_labels` or `config.hidden_size` classes.
summary_first_dropout (`float`, *optional*, defaults to 0.1):
- Argument used when doing sequence summary, used in the models
- [`OpenAIGPTDoubleHeadsModel`] and [`OpenAIGPTDoubleHeadsModel`].
+ Argument used when doing sequence summary, used in the models [`OpenAIGPTDoubleHeadsModel`] and
+ [`OpenAIGPTDoubleHeadsModel`].
The dropout ratio to be used after the projection and activation.
use_cache (`bool`, *optional*, defaults to `True`):
diff --git a/src/transformers/models/openai/modeling_openai.py b/src/transformers/models/openai/modeling_openai.py
index e9ceeb6af4..8bce94ffce 100644
--- a/src/transformers/models/openai/modeling_openai.py
+++ b/src/transformers/models/openai/modeling_openai.py
@@ -312,12 +312,13 @@ class OpenAIGPTDoubleHeadsModelOutput(ModelOutput):
mc_logits (`torch.FloatTensor` of shape `(batch_size, num_choices)`):
Prediction scores of the multiple choice classification head (scores for each choice before SoftMax).
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -333,19 +334,18 @@ class OpenAIGPTDoubleHeadsModelOutput(ModelOutput):
OPENAI_GPT_START_DOCSTRING = r"""
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
Parameters:
config ([`OpenAIGPTConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
OPENAI_GPT_INPUTS_DOCSTRING = r"""
@@ -353,9 +353,8 @@ OPENAI_GPT_INPUTS_DOCSTRING = r"""
input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`OpenAIGPTTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`OpenAIGPTTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -366,14 +365,16 @@ OPENAI_GPT_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
@@ -383,9 +384,9 @@ OPENAI_GPT_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -570,8 +571,8 @@ class OpenAIGPTLMHeadModel(OpenAIGPTPreTrainedModel):
r"""
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
- `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to
- `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
+ `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
+ are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -656,14 +657,15 @@ class OpenAIGPTDoubleHeadsModel(OpenAIGPTPreTrainedModel):
):
r"""
mc_token_ids (`torch.LongTensor` of shape `(batch_size, num_choices)`, *optional*, default to index of the last token of the input):
- Index of the classification token in each input sequence. Selected in the range `[0, input_ids.size(-1) - 1]`.
+ Index of the classification token in each input sequence. Selected in the range `[0, input_ids.size(-1) -
+ 1]`.
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
- `labels = input_ids` Indices are selected in `[-1, 0, ..., config.vocab_size]` All labels set to
- `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
+ `labels = input_ids` Indices are selected in `[-1, 0, ..., config.vocab_size]` All labels set to `-100` are
+ ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
mc_labels (`torch.LongTensor` of shape `(batch_size)`, *optional*):
- Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices]` where *num_choices* is the size of the second dimension of the input tensors. (see
- *input_ids* above)
+ Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices]`
+ where *num_choices* is the size of the second dimension of the input tensors. (see *input_ids* above)
Return:
@@ -733,12 +735,12 @@ class OpenAIGPTDoubleHeadsModel(OpenAIGPTPreTrainedModel):
@add_start_docstrings(
"""
The Original OpenAI GPT Model transformer with a sequence classification head on top (linear layer).
- [`OpenAIGPTForSequenceClassification`] uses the last token in order to do the classification, as
- other causal models (e.g. GPT-2) do. Since it does classification on the last token, it requires to know the
- position of the last token. If a `pad_token_id` is defined in the configuration, it finds the last token that
- is not a padding token in each row. If no `pad_token_id` is defined, it simply takes the last value in each
- row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of
- `input_ids`, it does the same (take the last value in each row of the batch).
+ [`OpenAIGPTForSequenceClassification`] uses the last token in order to do the classification, as other causal
+ models (e.g. GPT-2) do. Since it does classification on the last token, it requires to know the position of the
+ last token. If a `pad_token_id` is defined in the configuration, it finds the last token that is not a padding
+ token in each row. If no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since
+ it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take
+ the last value in each row of the batch).
""",
OPENAI_GPT_START_DOCSTRING,
)
@@ -774,8 +776,9 @@ class OpenAIGPTForSequenceClassification(OpenAIGPTPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
diff --git a/src/transformers/models/openai/modeling_tf_openai.py b/src/transformers/models/openai/modeling_tf_openai.py
index 2f31e5539a..88511f0c6b 100644
--- a/src/transformers/models/openai/modeling_tf_openai.py
+++ b/src/transformers/models/openai/modeling_tf_openai.py
@@ -398,12 +398,13 @@ class TFOpenAIGPTDoubleHeadsModelOutput(ModelOutput):
mc_logits (`tf.Tensor` of shape `(batch_size, num_choices)`):
Prediction scores of the multiple choice classification head (scores for each choice before SoftMax).
hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -417,13 +418,13 @@ class TFOpenAIGPTDoubleHeadsModelOutput(ModelOutput):
OPENAI_GPT_START_DOCSTRING = r"""
- This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading or saving, resizing the input
- embeddings, pruning heads etc.)
+ This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use
- it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage
- and behavior.
+ This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it
+ as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and
+ behavior.
@@ -432,11 +433,11 @@ OPENAI_GPT_START_DOCSTRING = r"""
- having all inputs as keyword arguments (like PyTorch models), or
- having all inputs as a list, tuple or dict in the first positional arguments.
- This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all
- the tensors in the first argument of the model call function: `model(inputs)`.
+ This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all the
+ tensors in the first argument of the model call function: `model(inputs)`.
- If you choose this second option, there are three possibilities you can use to gather all the input Tensors in
- the first positional argument :
+ If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the
+ first positional argument :
- a single Tensor with `input_ids` only and nothing else: `model(inputs_ids)`
- a list of varying length with one or several input Tensors IN THE ORDER given in the docstring:
@@ -449,8 +450,7 @@ OPENAI_GPT_START_DOCSTRING = r"""
Parameters:
config ([`OpenAIGPTConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
OPENAI_GPT_INPUTS_DOCSTRING = r"""
@@ -458,9 +458,8 @@ OPENAI_GPT_INPUTS_DOCSTRING = r"""
input_ids (`Numpy array` or `tf.Tensor` of shape `(batch_size, sequence_length)`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`OpenAIGPTTokenizer`]. See
- [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for
- details.
+ Indices can be obtained using [`OpenAIGPTTokenizer`]. See [`PreTrainedTokenizer.__call__`] and
+ [`PreTrainedTokenizer.encode`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`tf.Tensor` or `Numpy array` of shape `(batch_size, sequence_length)`, *optional*):
@@ -471,14 +470,16 @@ OPENAI_GPT_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`tf.Tensor` or `Numpy array` of shape `(batch_size, sequence_length)`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`tf.Tensor` or `Numpy array` of shape `(batch_size, sequence_length)`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
head_mask (`tf.Tensor` or `Numpy array` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
@@ -488,9 +489,9 @@ OPENAI_GPT_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
inputs_embeds (`tf.Tensor` or `Numpy array` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the
@@ -500,8 +501,8 @@ OPENAI_GPT_INPUTS_DOCSTRING = r"""
more detail. This argument can be used only in eager mode, in graph mode the value in the config will be
used instead.
return_dict (`bool`, *optional*):
- Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This
- argument can be used in eager mode, in graph mode the value will always be set to True.
+ Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This argument can be used
+ in eager mode, in graph mode the value will always be set to True.
training (`bool`, *optional*, defaults to `False`):
Whether or not to use the model in training mode (some modules like dropout modules have different
behaviors between training and evaluation).
@@ -617,7 +618,8 @@ class TFOpenAIGPTLMHeadModel(TFOpenAIGPTPreTrainedModel, TFCausalLanguageModelin
):
r"""
labels (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the cross entropy classification loss. Indices should be in `[0, ..., config.vocab_size - 1]`.
+ Labels for computing the cross entropy classification loss. Indices should be in `[0, ...,
+ config.vocab_size - 1]`.
"""
inputs = input_processing(
func=self.call,
@@ -713,7 +715,8 @@ class TFOpenAIGPTDoubleHeadsModel(TFOpenAIGPTPreTrainedModel):
):
r"""
mc_token_ids (`tf.Tensor` or `Numpy array` of shape `(batch_size, num_choices)`, *optional*, default to index of the last token of the input):
- Index of the classification token in each input sequence. Selected in the range `[0, input_ids.size(-1) - 1]`.
+ Index of the classification token in each input sequence. Selected in the range `[0, input_ids.size(-1) -
+ 1]`.
Return:
@@ -827,14 +830,14 @@ class TFOpenAIGPTDoubleHeadsModel(TFOpenAIGPTPreTrainedModel):
"""
The OpenAI GPT Model transformer with a sequence classification head on top (linear layer).
- [`TFOpenAIGPTForSequenceClassification`] uses the last token in order to do the classification,
- as other causal models (e.g. GPT-2) do.
+ [`TFOpenAIGPTForSequenceClassification`] uses the last token in order to do the classification, as other causal
+ models (e.g. GPT-2) do.
Since it does classification on the last token, it requires to know the position of the last token. If a
- `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each
- row. If no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot
- guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take
- the last value in each row of the batch).
+ `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
+ no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
+ padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
+ each row of the batch).
""",
OPENAI_GPT_START_DOCSTRING,
)
@@ -874,7 +877,8 @@ class TFOpenAIGPTForSequenceClassification(TFOpenAIGPTPreTrainedModel, TFSequenc
):
r"""
labels (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the cross entropy classification loss. Indices should be in `[0, ..., config.vocab_size - 1]`.
+ Labels for computing the cross entropy classification loss. Indices should be in `[0, ...,
+ config.vocab_size - 1]`.
"""
inputs = input_processing(
func=self.call,
diff --git a/src/transformers/models/openai/tokenization_openai.py b/src/transformers/models/openai/tokenization_openai.py
index 0b6987000c..af96fb06a9 100644
--- a/src/transformers/models/openai/tokenization_openai.py
+++ b/src/transformers/models/openai/tokenization_openai.py
@@ -78,8 +78,8 @@ class OpenAIGPTTokenizer(PreTrainedTokenizer):
- uses `SpaCy` tokenizer and `ftfy` for pre-BPE tokenization if they are installed, fallback to BERT's
`BasicTokenizer` if not.
- This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods.
- Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
+ this superclass for more information regarding those methods.
Args:
vocab_file (`str`):
diff --git a/src/transformers/models/openai/tokenization_openai_fast.py b/src/transformers/models/openai/tokenization_openai_fast.py
index 88bd569ba8..2df26c3a2f 100644
--- a/src/transformers/models/openai/tokenization_openai_fast.py
+++ b/src/transformers/models/openai/tokenization_openai_fast.py
@@ -45,8 +45,8 @@ class OpenAIGPTTokenizerFast(PreTrainedTokenizerFast):
- lower case all inputs
- uses BERT's BasicTokenizer for pre-BPE tokenization
- This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main
- methods. Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should
+ refer to this superclass for more information regarding those methods.
Args:
vocab_file (`str`):
diff --git a/src/transformers/models/pegasus/configuration_pegasus.py b/src/transformers/models/pegasus/configuration_pegasus.py
index 884a4524b5..91ce7c35ae 100644
--- a/src/transformers/models/pegasus/configuration_pegasus.py
+++ b/src/transformers/models/pegasus/configuration_pegasus.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" PEGASUS model configuration """
+""" PEGASUS model configuration"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging
@@ -28,19 +28,19 @@ PEGASUS_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class PegasusConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`PegasusModel`]. It is used to
- instantiate an PEGASUS model according to the specified arguments, defining the model architecture. Instantiating a
- configuration with the defaults will yield a similar configuration to that of the PEGASUS [google/pegasus-large](https://huggingface.co/google/pegasus-large) architecture.
+ This is the configuration class to store the configuration of a [`PegasusModel`]. It is used to instantiate an
+ PEGASUS model according to the specified arguments, defining the model architecture. Instantiating a configuration
+ with the defaults will yield a similar configuration to that of the PEGASUS
+ [google/pegasus-large](https://huggingface.co/google/pegasus-large) architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
vocab_size (`int`, *optional*, defaults to 50265):
Vocabulary size of the PEGASUS model. Defines the number of different tokens that can be represented by the
- `inputs_ids` passed when calling [`PegasusModel`] or
- [`TFPegasusModel`].
+ `inputs_ids` passed when calling [`PegasusModel`] or [`TFPegasusModel`].
d_model (`int`, *optional*, defaults to 1024):
Dimensionality of the layers and the pooler layer.
encoder_layers (`int`, *optional*, defaults to 12):
@@ -56,8 +56,8 @@ class PegasusConfig(PretrainedConfig):
encoder_ffn_dim (`int`, *optional*, defaults to 4096):
Dimensionality of the "intermediate" (often named feed-forward) layer in decoder.
activation_function (`str` or `function`, *optional*, defaults to `"gelu"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"silu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"silu"` and `"gelu_new"` are supported.
dropout (`float`, *optional*, defaults to 0.1):
The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
attention_dropout (`float`, *optional*, defaults to 0.0):
@@ -72,11 +72,11 @@ class PegasusConfig(PretrainedConfig):
init_std (`float`, *optional*, defaults to 0.02):
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
encoder_layerdrop: (`float`, *optional*, defaults to 0.0):
- The LayerDrop probability for the encoder. See the [LayerDrop paper](see
- https://arxiv.org/abs/1909.11556) for more details.
+ The LayerDrop probability for the encoder. See the [LayerDrop paper](see https://arxiv.org/abs/1909.11556)
+ for more details.
decoder_layerdrop: (`float`, *optional*, defaults to 0.0):
- The LayerDrop probability for the decoder. See the [LayerDrop paper](see
- https://arxiv.org/abs/1909.11556) for more details.
+ The LayerDrop probability for the decoder. See the [LayerDrop paper](see https://arxiv.org/abs/1909.11556)
+ for more details.
scale_embedding (`bool`, *optional*, defaults to `False`):
Scale embeddings by diving by sqrt(d_model).
use_cache (`bool`, *optional*, defaults to `True`):
diff --git a/src/transformers/models/pegasus/modeling_flax_pegasus.py b/src/transformers/models/pegasus/modeling_flax_pegasus.py
index c6f72cbc47..e11f7e2fa7 100644
--- a/src/transformers/models/pegasus/modeling_flax_pegasus.py
+++ b/src/transformers/models/pegasus/modeling_flax_pegasus.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" Flax PEGASUS model. """
+""" Flax PEGASUS model."""
import math
@@ -58,12 +58,13 @@ _CONFIG_FOR_DOC = "PegasusConfig"
_TOKENIZER_FOR_DOC = "PegasusTokenizer"
PEGASUS_START_DOCSTRING = r"""
- This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading or saving, resizing the input
- embeddings, pruning heads etc.)
+ This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a Flax Linen [flax.nn.Module](https://flax.readthedocs.io/en/latest/_autosummary/flax.nn.module.html) subclass. Use it as a regular Flax
- Module and refer to the Flax documentation for all matter related to general usage and behavior.
+ This model is also a Flax Linen
+ [flax.nn.Module](https://flax.readthedocs.io/en/latest/_autosummary/flax.nn.module.html) subclass. Use it as a
+ regular Flax Module and refer to the Flax documentation for all matter related to general usage and behavior.
Finally, this model supports inherent JAX features such as:
@@ -75,11 +76,10 @@ PEGASUS_START_DOCSTRING = r"""
Parameters:
config ([`PegasusConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the
- model weights.
+ configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the model weights.
dtype (`jax.numpy.dtype`, *optional*, defaults to `jax.numpy.float32`):
- The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on
- GPUs) and `jax.numpy.bfloat16` (on TPUs).
+ The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on GPUs) and
+ `jax.numpy.bfloat16` (on TPUs).
This can be used to enable mixed-precision training or half-precision inference on GPUs or TPUs. If
specified all the computation will be performed with the given `dtype`.
@@ -87,8 +87,8 @@ PEGASUS_START_DOCSTRING = r"""
**Note that this only specifies the dtype of the computation and does not influence the dtype of model
parameters.**
- If you wish to change the dtype of the model parameters, see
- [`~FlaxPreTrainedModel.to_fp16`] and [`~FlaxPreTrainedModel.to_bf16`].
+ If you wish to change the dtype of the model parameters, see [`~FlaxPreTrainedModel.to_fp16`] and
+ [`~FlaxPreTrainedModel.to_bf16`].
"""
PEGASUS_INPUTS_DOCSTRING = r"""
@@ -97,9 +97,8 @@ PEGASUS_INPUTS_DOCSTRING = r"""
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
it.
- Indices can be obtained using [`PegasusTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`PegasusTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`jnp.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
@@ -112,18 +111,19 @@ PEGASUS_INPUTS_DOCSTRING = r"""
decoder_input_ids (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`, *optional*):
Indices of decoder input sequence tokens in the vocabulary.
- Indices can be obtained using [`PegasusTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`PegasusTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are decoder input IDs?](../glossary#decoder-input-ids)
decoder_attention_mask (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`, *optional*):
- Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will
- also be used by default.
+ Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
+ be used by default.
- If you want to change padding behavior, you should modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy.
+ If you want to change padding behavior, you should modify to your needs. See diagram 1 in [the
+ paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy.
position_ids (`numpy.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
decoder_position_ids (`numpy.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the
range `[0, config.max_position_embeddings - 1]`.
@@ -144,9 +144,8 @@ PEGASUS_ENCODE_INPUTS_DOCSTRING = r"""
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
it.
- Indices can be obtained using [`PegasusTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`PegasusTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`jnp.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
@@ -157,7 +156,8 @@ PEGASUS_ENCODE_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
position_ids (`numpy.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -173,16 +173,14 @@ PEGASUS_DECODE_INPUTS_DOCSTRING = r"""
decoder_input_ids (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`):
Indices of decoder input sequence tokens in the vocabulary.
- Indices can be obtained using [`PegasusTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`PegasusTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are decoder input IDs?](../glossary#decoder-input-ids)
encoder_outputs (`tuple(tuple(jnp.ndarray)`):
- Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*:
- `attentions`) `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`,
- *optional*) is a sequence of hidden-states at the output of the last layer of the encoder. Used in the
- cross-attention of the decoder.
+ Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`)
+ `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of
+ hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder.
encoder_attention_mask (`jnp.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
@@ -191,10 +189,11 @@ PEGASUS_DECODE_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
decoder_attention_mask (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`, *optional*):
- Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will
- also be used by default.
+ Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
+ be used by default.
- If you want to change padding behavior, you should modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy.
+ If you want to change padding behavior, you should modify to your needs. See diagram 1 in [the
+ paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy.
decoder_position_ids (`numpy.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the
range `[0, config.max_position_embeddings - 1]`.
@@ -941,9 +940,10 @@ class FlaxPegasusPreTrainedModel(FlaxPreTrainedModel):
maximum possible length for auto-regressive decoding. Defines the sequence length of the initialized
cache.
encoder_outputs (`Union[FlaxBaseModelOutput, tuple(tuple(jnp.ndarray)]`):
- `encoder_outputs` consists of (`last_hidden_state`, *optional*: `hidden_states`,
- *optional*: `attentions`). `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of hidden-states at the output of the last layer of the
- encoder. Used in the cross-attention of the decoder.
+ `encoder_outputs` consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*:
+ `attentions`). `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*)
+ is a sequence of hidden-states at the output of the last layer of the encoder. Used in the
+ cross-attention of the decoder.
"""
# init input variables to retrieve cache
decoder_input_ids = jnp.ones((batch_size, max_length), dtype="i4")
@@ -1484,29 +1484,26 @@ FLAX_PEGASUS_CONDITIONAL_GENERATION_DOCSTRING = """
>>> from transformers import PegasusTokenizer, FlaxPegasusForConditionalGeneration
- >>> model = FlaxPegasusForConditionalGeneration.from_pretrained('google/pegasus-large')
- >>> tokenizer = PegasusTokenizer.from_pretrained('google/pegasus-large')
+ >>> model = FlaxPegasusForConditionalGeneration.from_pretrained('google/pegasus-large') >>> tokenizer =
+ PegasusTokenizer.from_pretrained('google/pegasus-large')
- >>> ARTICLE_TO_SUMMARIZE = "My friends are cool but they eat too many carbs."
- >>> inputs = tokenizer([ARTICLE_TO_SUMMARIZE], max_length=1024, return_tensors='np')
+ >>> ARTICLE_TO_SUMMARIZE = "My friends are cool but they eat too many carbs." >>> inputs =
+ tokenizer([ARTICLE_TO_SUMMARIZE], max_length=1024, return_tensors='np')
- >>> # Generate Summary
- >>> summary_ids = model.generate(inputs['input_ids']).sequences
- >>> print(tokenizer.batch_decode(summary_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False))
+ >>> # Generate Summary >>> summary_ids = model.generate(inputs['input_ids']).sequences >>>
+ print(tokenizer.batch_decode(summary_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False))
Mask filling example::
- >>> from transformers import PegasusTokenizer, FlaxPegasusForConditionalGeneration
- >>> tokenizer = PegasusTokenizer.from_pretrained('google/pegasus-large')
- >>> TXT = "My friends are but they eat too many carbs."
+ >>> from transformers import PegasusTokenizer, FlaxPegasusForConditionalGeneration >>> tokenizer =
+ PegasusTokenizer.from_pretrained('google/pegasus-large') >>> TXT = "My friends are but they eat too many
+ carbs."
- >>> model = FlaxPegasusForConditionalGeneration.from_pretrained('google/pegasus-large')
- >>> input_ids = tokenizer([TXT], return_tensors='np')['input_ids']
- >>> logits = model(input_ids).logits
+ >>> model = FlaxPegasusForConditionalGeneration.from_pretrained('google/pegasus-large') >>> input_ids =
+ tokenizer([TXT], return_tensors='np')['input_ids'] >>> logits = model(input_ids).logits
- >>> masked_index = (input_ids[0] == tokenizer.mask_token_id).nonzero().item()
- >>> probs = jax.nn.softmax(logits[0, masked_index], axis=0)
- >>> values, predictions = jax.lax.top_k(probs)
+ >>> masked_index = (input_ids[0] == tokenizer.mask_token_id).nonzero().item() >>> probs =
+ jax.nn.softmax(logits[0, masked_index], axis=0) >>> values, predictions = jax.lax.top_k(probs)
>>> tokenizer.decode(predictions).split()
"""
diff --git a/src/transformers/models/pegasus/modeling_pegasus.py b/src/transformers/models/pegasus/modeling_pegasus.py
index f4d4d3ca79..bcce420329 100755
--- a/src/transformers/models/pegasus/modeling_pegasus.py
+++ b/src/transformers/models/pegasus/modeling_pegasus.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" PyTorch PEGASUS model. """
+""" PyTorch PEGASUS model."""
import copy
import math
@@ -399,7 +399,8 @@ class PegasusDecoderLayer(nn.Module):
hidden_states (`torch.FloatTensor`): input to the layer of shape *(seq_len, batch, embed_dim)*
attention_mask (`torch.FloatTensor`): attention mask of size
*(batch, 1, tgt_len, src_len)* where padding elements are indicated by very large negative values.
- encoder_hidden_states (`torch.FloatTensor`): cross attention input to the layer of shape *(seq_len, batch, embed_dim)*
+ encoder_hidden_states (`torch.FloatTensor`):
+ cross attention input to the layer of shape *(seq_len, batch, embed_dim)*
encoder_attention_mask (`torch.FloatTensor`): encoder attention mask of size
*(batch, 1, tgt_len, src_len)* where padding elements are indicated by very large negative values.
layer_head_mask (`torch.FloatTensor`): mask for attention heads in a given layer of size
@@ -495,13 +496,13 @@ class PegasusPreTrainedModel(PreTrainedModel):
PEGASUS_START_DOCSTRING = r"""
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
Parameters:
config ([`PegasusConfig`]):
@@ -515,19 +516,16 @@ PEGASUS_GENERATION_EXAMPLE = r"""
>>> from transformers import PegasusTokenizer, PegasusForConditionalGeneration
- >>> model = PegasusForConditionalGeneration.from_pretrained('google/pegasus-xsum')
- >>> tokenizer = PegasusTokenizer.from_pretrained('google/pegasus-xsum')
+ >>> model = PegasusForConditionalGeneration.from_pretrained('google/pegasus-xsum') >>> tokenizer =
+ PegasusTokenizer.from_pretrained('google/pegasus-xsum')
- >>> ARTICLE_TO_SUMMARIZE = (
- ... "PG&E stated it scheduled the blackouts in response to forecasts for high winds "
- ... "amid dry conditions. The aim is to reduce the risk of wildfires. Nearly 800 thousand customers were "
- ... "scheduled to be affected by the shutoffs which were expected to last through at least midday tomorrow."
- ... )
- >>> inputs = tokenizer([ARTICLE_TO_SUMMARIZE], max_length=1024, return_tensors='pt')
+ >>> ARTICLE_TO_SUMMARIZE = ( ... "PG&E stated it scheduled the blackouts in response to forecasts for high
+ winds " ... "amid dry conditions. The aim is to reduce the risk of wildfires. Nearly 800 thousand customers
+ were " ... "scheduled to be affected by the shutoffs which were expected to last through at least midday
+ tomorrow." ... ) >>> inputs = tokenizer([ARTICLE_TO_SUMMARIZE], max_length=1024, return_tensors='pt')
- >>> # Generate Summary
- >>> summary_ids = model.generate(inputs['input_ids'])
- >>> print([tokenizer.decode(g, skip_special_tokens=True, clean_up_tokenization_spaces=False) for g in summary_ids])
+ >>> # Generate Summary >>> summary_ids = model.generate(inputs['input_ids']) >>> print([tokenizer.decode(g,
+ skip_special_tokens=True, clean_up_tokenization_spaces=False) for g in summary_ids])
"""
PEGASUS_INPUTS_DOCSTRING = r"""
@@ -536,9 +534,8 @@ PEGASUS_INPUTS_DOCSTRING = r"""
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
it.
- Indices can be obtained using [`PegasusTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`PegasusTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -551,9 +548,8 @@ PEGASUS_INPUTS_DOCSTRING = r"""
decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
Indices of decoder input sequence tokens in the vocabulary.
- Indices can be obtained using [`PegasusTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`PegasusTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are decoder input IDs?](../glossary#decoder-input-ids)
@@ -561,8 +557,8 @@ PEGASUS_INPUTS_DOCSTRING = r"""
`past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
`past_key_values`).
decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
- Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will
- also be used by default.
+ Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
+ be used by default.
head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*):
Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`:
@@ -576,39 +572,42 @@ PEGASUS_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*):
- Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, 1]`:
+ Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0,
+ 1]`:
- 1 indicates the head is **not masked**,
- 0 indicates the head is **masked**.
encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*):
- Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*:
- `attentions`) `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`,
- *optional*) is a sequence of hidden-states at the output of the last layer of the encoder. Used in the
- cross-attention of the decoder.
+ Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`)
+ `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of
+ hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder.
past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors
- of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
- shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
+ Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
+ `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape
+ `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all ``decoder_input_ids``` of shape `(batch_size, sequence_length)`. inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ ``decoder_input_ids``` of shape `(batch_size, sequence_length)`. inputs_embeds (`torch.FloatTensor` of
+ shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids`
+ you can choose to directly pass an embedded representation. This is useful if you want more control over
+ how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup
+ matrix.
decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*):
Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded
- representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds`
- have to be input (see `past_key_values`). This is useful if you want more control over how to convert
+ representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be
+ input (see `past_key_values`). This is useful if you want more control over how to convert
`decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix.
- If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds`
- takes the value of `inputs_embeds`.
+ If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds` takes the value
+ of `inputs_embeds`.
use_cache (`bool`, *optional*):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`).
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`).
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -660,7 +659,8 @@ class PegasusEncoder(PegasusPreTrainedModel):
def resize_position_embeddings(self, new_num_position_embeddings: int):
"""
- Resizes position embeddings matrix of the model if `new_num_position_embeddings != config.max_position_embeddings`.
+ Resizes position embeddings matrix of the model if `new_num_position_embeddings !=
+ config.max_position_embeddings`.
Arguments:
new_num_position_embeddings (`int`):
@@ -702,9 +702,8 @@ class PegasusEncoder(PegasusPreTrainedModel):
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
provide it.
- Indices can be obtained using [`PegasusTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`]
- for details.
+ Indices can be obtained using [`PegasusTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -721,9 +720,9 @@ class PegasusEncoder(PegasusPreTrainedModel):
- 0 indicates the head is **masked**.
inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded
- representation. This is useful if you want more control over how to convert `input_ids` indices
- into associated vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
+ This is useful if you want more control over how to convert `input_ids` indices into associated vectors
+ than the model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under
returned tensors for more detail.
@@ -880,7 +879,8 @@ class PegasusDecoder(PegasusPreTrainedModel):
def resize_position_embeddings(self, new_num_position_embeddings: int):
"""
- Resizes position embeddings matrix of the model if `new_num_position_embeddings != config.max_position_embeddings`.
+ Resizes position embeddings matrix of the model if `new_num_position_embeddings !=
+ config.max_position_embeddings`.
Arguments:
new_num_position_embeddings (`int`):
@@ -927,9 +927,8 @@ class PegasusDecoder(PegasusPreTrainedModel):
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
provide it.
- Indices can be obtained using [`PegasusTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`]
- for details.
+ Indices can be obtained using [`PegasusTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -964,19 +963,20 @@ class PegasusDecoder(PegasusPreTrainedModel):
- 0 indicates the head is **masked**.
past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2
- tensors of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional
- tensors of shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
+ Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of
+ shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
+ shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
Contains pre-computed hidden-states (key and values in the self-attention blocks and in the
- cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential
- decoding.
+ cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
- If `past_key_values` are used, the user can optionally input only the last
- `decoder_input_ids` (those that don't have their past key value states given to this model) of
- shape `(batch_size, 1)` instead of all ``decoder_input_ids``` of shape `(batch_size,
- sequence_length)`. inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices
- into associated vectors than the model's internal embedding lookup matrix.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those
+ that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of
+ all ``decoder_input_ids``` of shape `(batch_size, sequence_length)`. inputs_embeds (`torch.FloatTensor`
+ of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing
+ `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more
+ control over how to convert `input_ids` indices into associated vectors than the model's internal
+ embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under
returned tensors for more detail.
@@ -1154,7 +1154,8 @@ class PegasusModel(PegasusPreTrainedModel):
def resize_position_embeddings(self, new_num_position_embeddings: int):
"""
- Resizes position embeddings matrix of the model if `new_num_position_embeddings != config.max_position_embeddings`.
+ Resizes position embeddings matrix of the model if `new_num_position_embeddings !=
+ config.max_position_embeddings`.
Arguments:
new_num_position_embeddings (`int`):
@@ -1318,7 +1319,8 @@ class PegasusForConditionalGeneration(PegasusPreTrainedModel):
def resize_position_embeddings(self, new_num_position_embeddings: int):
"""
- Resizes position embeddings matrix of the model if `new_num_position_embeddings != config.max_position_embeddings`.
+ Resizes position embeddings matrix of the model if `new_num_position_embeddings !=
+ config.max_position_embeddings`.
Arguments:
new_num_position_embeddings (`int`):
@@ -1362,7 +1364,8 @@ class PegasusForConditionalGeneration(PegasusPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
+ Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
+ config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
(masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
Returns:
@@ -1512,7 +1515,8 @@ class PegasusForCausalLM(PegasusPreTrainedModel):
def resize_position_embeddings(self, new_num_position_embeddings: int):
"""
- Resizes position embeddings matrix of the model if `new_num_position_embeddings != config.max_position_embeddings`.
+ Resizes position embeddings matrix of the model if `new_num_position_embeddings !=
+ config.max_position_embeddings`.
Arguments:
new_num_position_embeddings (`int`):
@@ -1549,9 +1553,8 @@ class PegasusForCausalLM(PegasusPreTrainedModel):
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
provide it.
- Indices can be obtained using [`PegasusTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`]
- for details.
+ Indices can be obtained using [`PegasusTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -1580,25 +1583,24 @@ class PegasusForCausalLM(PegasusPreTrainedModel):
- 0 indicates the head is **masked**.
past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2
- tensors of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional
- tensors of shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. The two
- additional tensors are only required when the model is used as a decoder in a Sequence to Sequence
- model.
+ Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of
+ shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
+ shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. The two additional
+ tensors are only required when the model is used as a decoder in a Sequence to Sequence model.
Contains pre-computed hidden-states (key and values in the self-attention blocks and in the
- cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential
- decoding.
+ cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those
+ that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of
+ all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are
- ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
+ Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
+ config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
+ (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
use_cache (`bool`, *optional*):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`).
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
+ (see `past_key_values`).
- 1 for tokens that are **not masked**,
- 0 for tokens that are **masked**.
diff --git a/src/transformers/models/pegasus/modeling_tf_pegasus.py b/src/transformers/models/pegasus/modeling_tf_pegasus.py
index d0496dc215..0da2a9aa92 100644
--- a/src/transformers/models/pegasus/modeling_tf_pegasus.py
+++ b/src/transformers/models/pegasus/modeling_tf_pegasus.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" TF 2.0 Pegasus model. """
+""" TF 2.0 Pegasus model."""
import random
@@ -420,7 +420,8 @@ class TFPegasusDecoderLayer(tf.keras.layers.Layer):
hidden_states (`tf.Tensor`): input to the layer of shape *(seq_len, batch, embed_dim)*
attention_mask (`tf.Tensor`): attention mask of size
*(batch, 1, tgt_len, src_len)* where padding elements are indicated by very large negative values.
- encoder_hidden_states (`tf.Tensor`): cross attention input to the layer of shape *(seq_len, batch, embed_dim)*
+ encoder_hidden_states (`tf.Tensor`):
+ cross attention input to the layer of shape *(seq_len, batch, embed_dim)*
encoder_attention_mask (`tf.Tensor`): encoder attention mask of size
*(batch, 1, tgt_len, src_len)* where padding elements are indicated by very large negative values.
layer_head_mask (`tf.Tensor`): mask for attention heads in a given layer of size
@@ -518,13 +519,13 @@ class TFPegasusPreTrainedModel(TFPreTrainedModel):
PEGASUS_START_DOCSTRING = r"""
- This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading or saving, resizing the input
- embeddings, pruning heads etc.)
+ This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use
- it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage
- and behavior.
+ This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it
+ as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and
+ behavior.
@@ -533,11 +534,11 @@ PEGASUS_START_DOCSTRING = r"""
- having all inputs as keyword arguments (like PyTorch models), or
- having all inputs as a list, tuple or dict in the first positional arguments.
- This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all
- the tensors in the first argument of the model call function: `model(inputs)`.
+ This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all the
+ tensors in the first argument of the model call function: `model(inputs)`.
- If you choose this second option, there are three possibilities you can use to gather all the input Tensors in
- the first positional argument :
+ If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the
+ first positional argument :
- a single Tensor with `input_ids` only and nothing else: `model(input_ids)`
- a list of varying length with one or several input Tensors IN THE ORDER given in the docstring:
@@ -550,8 +551,7 @@ PEGASUS_START_DOCSTRING = r"""
Args:
config ([`PegasusConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~TFPreTrainedModel.from_pretrained`] method to load the
- model weights.
+ configuration. Check out the [`~TFPreTrainedModel.from_pretrained`] method to load the model weights.
"""
PEGASUS_GENERATION_EXAMPLE = r"""
@@ -559,19 +559,16 @@ PEGASUS_GENERATION_EXAMPLE = r"""
>>> from transformers import PegasusTokenizer, TFPegasusForConditionalGeneration
- >>> model = TFPegasusForConditionalGeneration.from_pretrained('google/pegasus-xsum')
- >>> tokenizer = PegasusTokenizer.from_pretrained('google/pegasus-xsum')
+ >>> model = TFPegasusForConditionalGeneration.from_pretrained('google/pegasus-xsum') >>> tokenizer =
+ PegasusTokenizer.from_pretrained('google/pegasus-xsum')
- >>> ARTICLE_TO_SUMMARIZE = (
- ... "PG&E stated it scheduled the blackouts in response to forecasts for high winds "
- ... "amid dry conditions. The aim is to reduce the risk of wildfires. Nearly 800 thousand customers were "
- ... "scheduled to be affected by the shutoffs which were expected to last through at least midday tomorrow."
- ... )
- >>> inputs = tokenizer([ARTICLE_TO_SUMMARIZE], max_length=1024, return_tensors='tf')
+ >>> ARTICLE_TO_SUMMARIZE = ( ... "PG&E stated it scheduled the blackouts in response to forecasts for high
+ winds " ... "amid dry conditions. The aim is to reduce the risk of wildfires. Nearly 800 thousand customers
+ were " ... "scheduled to be affected by the shutoffs which were expected to last through at least midday
+ tomorrow." ... ) >>> inputs = tokenizer([ARTICLE_TO_SUMMARIZE], max_length=1024, return_tensors='tf')
- >>> # Generate Summary
- >>> summary_ids = model.generate(inputs['input_ids'])
- >>> print([tokenizer.decode(g, skip_special_tokens=True, clean_up_tokenization_spaces=False) for g in summary_ids])
+ >>> # Generate Summary >>> summary_ids = model.generate(inputs['input_ids']) >>> print([tokenizer.decode(g,
+ skip_special_tokens=True, clean_up_tokenization_spaces=False) for g in summary_ids])
"""
PEGASUS_INPUTS_DOCSTRING = r"""
@@ -579,9 +576,8 @@ PEGASUS_INPUTS_DOCSTRING = r"""
input_ids (`tf.Tensor` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`PegasusTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`PegasusTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`tf.Tensor` of shape `({0})`, *optional*):
@@ -594,9 +590,8 @@ PEGASUS_INPUTS_DOCSTRING = r"""
decoder_input_ids (`tf.Tensor` of shape `(batch_size, target_sequence_length)`, *optional*):
Indices of decoder input sequence tokens in the vocabulary.
- Indices can be obtained using [`PegasusTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`PegasusTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are decoder input IDs?](../glossary#decoder-input-ids)
@@ -628,15 +623,15 @@ PEGASUS_INPUTS_DOCSTRING = r"""
of shape `(batch_size, sequence_length, hidden_size)` is a sequence of
past_key_values (`Tuple[Tuple[tf.Tensor]]` of length `config.n_layers`)
contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ `decoder_input_ids` of shape `(batch_size, sequence_length)`.
use_cache (`bool`, *optional*, defaults to `True`):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`). Set to `False` during training, `True` during generation
- output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all
- attention layers. See `attentions` under returned tensors for more detail. This argument can be used only
- in eager mode, in graph mode the value in the config will be used instead.
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`). Set to `False` during training, `True` during generation output_attentions (`bool`,
+ *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions`
+ under returned tensors for more detail. This argument can be used only in eager mode, in graph mode the
+ value in the config will be used instead.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the
@@ -646,8 +641,8 @@ PEGASUS_INPUTS_DOCSTRING = r"""
more detail. This argument can be used only in eager mode, in graph mode the value in the config will be
used instead.
return_dict (`bool`, *optional*):
- Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This
- argument can be used in eager mode, in graph mode the value will always be set to True.
+ Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This argument can be used
+ in eager mode, in graph mode the value will always be set to True.
training (`bool`, *optional*, defaults to `False`):
Whether or not to use the model in training mode (some modules like dropout modules have different
behaviors between training and evaluation).
@@ -707,9 +702,8 @@ class TFPegasusEncoder(tf.keras.layers.Layer):
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
provide it.
- Indices can be obtained using [`PegasusTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`]
- for details.
+ Indices can be obtained using [`PegasusTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -719,15 +713,16 @@ class TFPegasusEncoder(tf.keras.layers.Layer):
- 0 for tokens that are **masked**.
[What are attention masks?](../glossary#attention-mask)
- head_mask (`tf.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, `optional): Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`:
+ head_mask (`tf.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, `optional):
+ Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`:
- 1 indicates the head is **not masked**,
- 0 indicates the head is **masked**.
inputs_embeds (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded
- representation. This is useful if you want more control over how to convert `input_ids` indices
- into associated vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
+ This is useful if you want more control over how to convert `input_ids` indices into associated vectors
+ than the model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under
returned tensors for more detail. This argument can be used only in eager mode, in graph mode the value
@@ -737,8 +732,8 @@ class TFPegasusEncoder(tf.keras.layers.Layer):
for more detail. This argument can be used only in eager mode, in graph mode the value in the config
will be used instead.
return_dict (`bool`, *optional*):
- Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This
- argument can be used in eager mode, in graph mode the value will always be set to True.
+ Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This argument can be
+ used in eager mode, in graph mode the value will always be set to True.
training (`bool`, *optional*, defaults to `False`):
Whether or not to use the model in training mode (some modules like dropout modules have different
behaviors between training and evaluation).
@@ -881,9 +876,8 @@ class TFPegasusDecoder(tf.keras.layers.Layer):
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
provide it.
- Indices can be obtained using [`PegasusTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`]
- for details.
+ Indices can be obtained using [`PegasusTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -920,11 +914,13 @@ class TFPegasusDecoder(tf.keras.layers.Layer):
Contains precomputed key and value hidden-states of the attention blocks. Can be used to speed up
decoding.
- If `past_key_values` are used, the user can optionally input only the last
- `decoder_input_ids` (those that don't have their past key value states given to this model) of
- shape `(batch_size, 1)` instead of all ``decoder_input_ids``` of shape `(batch_size,
- sequence_length)`. inputs_embeds (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices
- into associated vectors than the model's internal embedding lookup matrix.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those
+ that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of
+ all ``decoder_input_ids``` of shape `(batch_size, sequence_length)`. inputs_embeds (`tf.Tensor` of
+ shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing
+ `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more
+ control over how to convert `input_ids` indices into associated vectors than the model's internal
+ embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under
returned tensors for more detail. This argument can be used only in eager mode, in graph mode the value
@@ -934,8 +930,8 @@ class TFPegasusDecoder(tf.keras.layers.Layer):
for more detail. This argument can be used only in eager mode, in graph mode the value in the config
will be used instead.
return_dict (`bool`, *optional*):
- Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This
- argument can be used in eager mode, in graph mode the value will always be set to True.
+ Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This argument can be
+ used in eager mode, in graph mode the value will always be set to True.
training (`bool`, *optional*, defaults to `False`):
Whether or not to use the model in training mode (some modules like dropout modules have different
behaviors between training and evaluation).
@@ -1383,7 +1379,8 @@ class TFPegasusForConditionalGeneration(TFPegasusPreTrainedModel, TFCausalLangua
):
"""
labels (`tf.tensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
+ Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
+ config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
(masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
Returns:
diff --git a/src/transformers/models/pegasus/tokenization_pegasus.py b/src/transformers/models/pegasus/tokenization_pegasus.py
index c582bc71bc..6b147ea05a 100644
--- a/src/transformers/models/pegasus/tokenization_pegasus.py
+++ b/src/transformers/models/pegasus/tokenization_pegasus.py
@@ -42,8 +42,8 @@ class PegasusTokenizer(PreTrainedTokenizer):
r"""
Construct a PEGASUS tokenizer. Based on [SentencePiece](https://github.com/google/sentencepiece).
- This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods.
- Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
+ this superclass for more information regarding those methods.
Args:
vocab_file (`str`):
@@ -56,8 +56,8 @@ class PegasusTokenizer(PreTrainedTokenizer):
- When building a sequence using special tokens, this is not the token that is used for the end of
- sequence. The token used is the `sep_token`.
+ When building a sequence using special tokens, this is not the token that is used for the end of sequence.
+ The token used is the `sep_token`.
@@ -80,7 +80,9 @@ class PegasusTokenizer(PreTrainedTokenizer):
tokenizer](https://github.com/google-research/pegasus/blob/939830367bcf411193d2b5eca2f2f90f3f9260ca/pegasus/ops/pretrain_parsing_ops.cc#L66)
that uses the tokens 2 - 104 only for pretraining
sp_model_kwargs (`dict`, *optional*):
- Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things, to set:
+ Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for
+ SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things,
+ to set:
- `enable_sampling`: Enable subword regularization.
- `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout.
diff --git a/src/transformers/models/pegasus/tokenization_pegasus_fast.py b/src/transformers/models/pegasus/tokenization_pegasus_fast.py
index 12d5fcaeed..67c065e7ec 100644
--- a/src/transformers/models/pegasus/tokenization_pegasus_fast.py
+++ b/src/transformers/models/pegasus/tokenization_pegasus_fast.py
@@ -51,10 +51,11 @@ PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
class PegasusTokenizerFast(PreTrainedTokenizerFast):
r"""
- Construct a "fast" PEGASUS tokenizer (backed by HuggingFace's *tokenizers* library). Based on [Unigram](https://huggingface.co/docs/tokenizers/python/latest/components.html?highlight=unigram#models).
+ Construct a "fast" PEGASUS tokenizer (backed by HuggingFace's *tokenizers* library). Based on
+ [Unigram](https://huggingface.co/docs/tokenizers/python/latest/components.html?highlight=unigram#models).
- This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main
- methods. Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should
+ refer to this superclass for more information regarding those methods.
Args:
vocab_file (`str`):
@@ -67,8 +68,8 @@ class PegasusTokenizerFast(PreTrainedTokenizerFast):
- When building a sequence using special tokens, this is not the token that is used for the end of
- sequence. The token used is the `sep_token`.
+ When building a sequence using special tokens, this is not the token that is used for the end of sequence.
+ The token used is the `sep_token`.
diff --git a/src/transformers/models/perceiver/configuration_perceiver.py b/src/transformers/models/perceiver/configuration_perceiver.py
index 79a6d60689..fdf1f01243 100644
--- a/src/transformers/models/perceiver/configuration_perceiver.py
+++ b/src/transformers/models/perceiver/configuration_perceiver.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" Perceiver model configuration """
+""" Perceiver model configuration"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging
@@ -28,13 +28,13 @@ PERCEIVER_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class PerceiverConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`PerceiverModel`]. It is used
- to instantiate an Perceiver model according to the specified arguments, defining the model architecture.
- Instantiating a configuration with the defaults will yield a similar configuration to that of the Perceiver
+ This is the configuration class to store the configuration of a [`PerceiverModel`]. It is used to instantiate an
+ Perceiver model according to the specified arguments, defining the model architecture. Instantiating a
+ configuration with the defaults will yield a similar configuration to that of the Perceiver
[deepmind/language-perceiver](https://huggingface.co/deepmind/language-perceiver) architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
num_latents (`int`, *optional*, defaults to 256):
@@ -65,8 +65,8 @@ class PerceiverConfig(PretrainedConfig):
cross_attention_widening_factor (`int`, *optional*, defaults to 1):
Dimension of the feed-forward layer in the self-attention layers of the Transformer encoder.
hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"selu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"selu"` and `"gelu_new"` are supported.
attention_probs_dropout_prob (`float`, *optional*, defaults to 0.1):
The dropout ratio for the attention probabilities.
initializer_range (`float`, *optional*, defaults to 0.02):
diff --git a/src/transformers/models/perceiver/feature_extraction_perceiver.py b/src/transformers/models/perceiver/feature_extraction_perceiver.py
index 49ca6d9cba..5ae9e1e4dc 100644
--- a/src/transformers/models/perceiver/feature_extraction_perceiver.py
+++ b/src/transformers/models/perceiver/feature_extraction_perceiver.py
@@ -38,26 +38,25 @@ class PerceiverFeatureExtractor(FeatureExtractionMixin, ImageFeatureExtractionMi
r"""
Constructs a Perceiver feature extractor.
- This feature extractor inherits from [`ImageFeatureExtractionMixin`] which contains most of the
- main methods. Users should refer to this superclass for more information regarding those methods.
+ This feature extractor inherits from [`ImageFeatureExtractionMixin`] which contains most of the main methods. Users
+ should refer to this superclass for more information regarding those methods.
Args:
do_center_crop (`bool`, *optional*, defaults to `True`):
- Whether to crop the input at the center. If the input size is smaller than `crop_size` along any edge,
- the image is padded with 0's and then center cropped.
+ Whether to crop the input at the center. If the input size is smaller than `crop_size` along any edge, the
+ image is padded with 0's and then center cropped.
crop_size (`int`, *optional*, defaults to 256):
- Desired output size when applying center-cropping. Only has an effect if `do_center_crop` is set to
- `True`.
+ Desired output size when applying center-cropping. Only has an effect if `do_center_crop` is set to `True`.
do_resize (`bool`, *optional*, defaults to `True`):
Whether to resize the input to a certain `size`.
size (`int` or `Tuple(int)`, *optional*, defaults to 224):
Resize the input to the given size. If a tuple is provided, it should be (width, height). If only an
- integer is provided, then the input will be resized to (size, size). Only has an effect if `do_resize`
- is set to `True`.
+ integer is provided, then the input will be resized to (size, size). Only has an effect if `do_resize` is
+ set to `True`.
resample (`int`, *optional*, defaults to `PIL.Image.BICUBIC`):
An optional resampling filter. This can be one of `PIL.Image.NEAREST`, `PIL.Image.BOX`,
- `PIL.Image.BILINEAR`, `PIL.Image.HAMMING`, `PIL.Image.BICUBIC` or `PIL.Image.LANCZOS`.
- Only has an effect if `do_resize` is set to `True`.
+ `PIL.Image.BILINEAR`, `PIL.Image.HAMMING`, `PIL.Image.BICUBIC` or `PIL.Image.LANCZOS`. Only has an effect
+ if `do_resize` is set to `True`.
do_normalize (`bool`, *optional*, defaults to `True`):
Whether or not to normalize the input with `image_mean` and `image_std`.
image_mean (`List[int]`, defaults to `[0.485, 0.456, 0.406]`):
@@ -92,8 +91,8 @@ class PerceiverFeatureExtractor(FeatureExtractionMixin, ImageFeatureExtractionMi
def center_crop(self, image):
"""
- Crops `image` to *self.crop_size* using a center crop. Note that if the image is too small to be cropped
- to the size given, it will be padded (so the returned result has the size asked).
+ Crops `image` to *self.crop_size* using a center crop. Note that if the image is too small to be cropped to the
+ size given, it will be padded (so the returned result has the size asked).
Args:
image (`PIL.Image.Image` or `np.ndarray` or `torch.Tensor`):
diff --git a/src/transformers/models/perceiver/modeling_perceiver.py b/src/transformers/models/perceiver/modeling_perceiver.py
index dce7c92af8..12b9749aba 100755
--- a/src/transformers/models/perceiver/modeling_perceiver.py
+++ b/src/transformers/models/perceiver/modeling_perceiver.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" PyTorch Perceiver model. """
+""" PyTorch Perceiver model."""
import abc
import math
@@ -73,15 +73,17 @@ class PerceiverModelOutput(ModelOutput):
last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
Sequence of hidden-states at the output of the last layer of the model.
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of
- each layer plus the initial embedding outputs.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer
+ plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the
- weighted average in the self-attention heads.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
+ the self-attention heads.
cross_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights of the decoder's cross-attention layer, after the
- attention softmax, used to compute the weighted average in the cross-attention heads.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`. Attentions weights of the decoder's cross-attention layer, after the attention softmax,
+ used to compute the weighted average in the cross-attention heads.
"""
logits: torch.FloatTensor = None
@@ -100,8 +102,9 @@ class PerceiverDecoderOutput(ModelOutput):
logits (`torch.FloatTensor` of shape `(batch_size, num_labels)`):
Output of the basic decoder.
cross_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights of the decoder's cross-attention layer, after the
- attention softmax, used to compute the weighted average in the cross-attention heads.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`. Attentions weights of the decoder's cross-attention layer, after the attention softmax,
+ used to compute the weighted average in the cross-attention heads.
"""
logits: torch.FloatTensor = None
@@ -119,15 +122,17 @@ class PerceiverMaskedLMOutput(ModelOutput):
logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of
- each layer plus the initial embedding outputs.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer
+ plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, num_latents, num_latents)`. Attentions weights after the attention softmax, used to compute the weighted average in the
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, num_latents,
+ num_latents)`. Attentions weights after the attention softmax, used to compute the weighted average in the
self-attention heads.
cross_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights of the decoder's cross-attention layer, after the
- attention softmax, used to compute the weighted average in the cross-attention heads.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`. Attentions weights of the decoder's cross-attention layer, after the attention softmax,
+ used to compute the weighted average in the cross-attention heads.
"""
loss: Optional[torch.FloatTensor] = None
@@ -149,15 +154,17 @@ class PerceiverClassifierOutput(ModelOutput):
logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
Classification (or regression if config.num_labels==1) scores (before SoftMax).
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of
- each layer plus the initial embedding outputs.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer
+ plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the
- weighted average in the self-attention heads.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
+ the self-attention heads.
cross_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights of the decoder's cross-attention layer, after the
- attention softmax, used to compute the weighted average in the cross-attention heads.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`. Attentions weights of the decoder's cross-attention layer, after the attention softmax,
+ used to compute the weighted average in the cross-attention heads.
"""
loss: Optional[torch.FloatTensor] = None
@@ -645,8 +652,7 @@ PERCEIVER_START_DOCSTRING = r"""
Parameters:
config ([`PerceiverConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
PERCEIVER_MODEL_START_DOCSTRING = r"""
@@ -657,8 +663,7 @@ PERCEIVER_MODEL_START_DOCSTRING = r"""
Parameters:
config ([`PerceiverConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
decoder (*DecoderType*, *optional*):
Optional decoder to use to decode the latent representation of the encoder. Examples include
*transformers.models.perceiver.modeling_perceiver.PerceiverBasicDecoder*,
@@ -937,7 +942,7 @@ class PerceiverModel(PerceiverPreTrainedModel):
)
-@add_start_docstrings("""Example use of Perceiver for masked language modeling. """, PERCEIVER_START_DOCSTRING)
+@add_start_docstrings("""Example use of Perceiver for masked language modeling.""", PERCEIVER_START_DOCSTRING)
class PerceiverForMaskedLM(PerceiverPreTrainedModel):
def __init__(self, config):
super().__init__(config)
@@ -984,8 +989,9 @@ class PerceiverForMaskedLM(PerceiverPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
Returns:
@@ -1063,7 +1069,7 @@ class PerceiverForMaskedLM(PerceiverPreTrainedModel):
)
-@add_start_docstrings("""Example use of Perceiver for text classification. """, PERCEIVER_START_DOCSTRING)
+@add_start_docstrings("""Example use of Perceiver for text classification.""", PERCEIVER_START_DOCSTRING)
class PerceiverForSequenceClassification(PerceiverPreTrainedModel):
def __init__(self, config):
super().__init__(config)
@@ -1100,8 +1106,9 @@ class PerceiverForSequenceClassification(PerceiverPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the classification/regression loss. Indices should be in `[0, ..., config.num_labels -
+ 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels >
+ 1` a classification loss is computed (Cross-Entropy).
Returns:
@@ -1179,11 +1186,10 @@ Example use of Perceiver for image classification, for tasks such as ImageNet.
This model uses learned position embeddings. In other words, this model is not given any privileged information about
the structure of images. As shown in the paper, this model can achieve a top-1 accuracy of 72.7 on ImageNet.
-[`PerceiverForImageClassificationLearned`] uses
-[`~models.perceiver.modeling_perceiver.PerceiverImagePreprocessor`] (with `prep_type="conv1x1"`)
-to preprocess the input images, and
-[`~models.perceiver.modeling_perceiver.PerceiverClassificationDecoder`] to decode the latent
-representation of [`PerceiverModel`] into classification logits.
+[`PerceiverForImageClassificationLearned`] uses [`~models.perceiver.modeling_perceiver.PerceiverImagePreprocessor`]
+(with `prep_type="conv1x1"`) to preprocess the input images, and
+[`~models.perceiver.modeling_perceiver.PerceiverClassificationDecoder`] to decode the latent representation of
+[`PerceiverModel`] into classification logits.
""",
PERCEIVER_START_DOCSTRING,
)
@@ -1233,8 +1239,9 @@ class PerceiverForImageClassificationLearned(PerceiverPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the image classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
Returns:
@@ -1318,11 +1325,10 @@ Example use of Perceiver for image classification, for tasks such as ImageNet.
This model uses fixed 2D Fourier position embeddings. As shown in the paper, this model can achieve a top-1 accuracy of
79.0 on ImageNet, and 84.5 when pre-trained on a large-scale dataset (i.e. JFT).
-[`PerceiverForImageClassificationLearned`] uses
-[`~models.perceiver.modeling_perceiver.PerceiverImagePreprocessor`] (with `prep_type="pixels"`)
-to preprocess the input images, and
-[`~models.perceiver.modeling_perceiver.PerceiverClassificationDecoder`] to decode the latent
-representation of [`PerceiverModel`] into classification logits.
+[`PerceiverForImageClassificationLearned`] uses [`~models.perceiver.modeling_perceiver.PerceiverImagePreprocessor`]
+(with `prep_type="pixels"`) to preprocess the input images, and
+[`~models.perceiver.modeling_perceiver.PerceiverClassificationDecoder`] to decode the latent representation of
+[`PerceiverModel`] into classification logits.
""",
PERCEIVER_START_DOCSTRING,
)
@@ -1370,8 +1376,9 @@ class PerceiverForImageClassificationFourier(PerceiverPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the image classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
Returns:
@@ -1454,11 +1461,10 @@ Example use of Perceiver for image classification, for tasks such as ImageNet.
This model uses a 2D conv+maxpool preprocessing network. As shown in the paper, this model can achieve a top-1 accuracy
of 82.1 on ImageNet.
-[`PerceiverForImageClassificationLearned`] uses
-[`~models.perceiver.modeling_perceiver.PerceiverImagePreprocessor`] (with `prep_type="conv"`) to
-preprocess the input images, and
-[`~models.perceiver.modeling_perceiver.PerceiverClassificationDecoder`] to decode the latent
-representation of [`PerceiverModel`] into classification logits.
+[`PerceiverForImageClassificationLearned`] uses [`~models.perceiver.modeling_perceiver.PerceiverImagePreprocessor`]
+(with `prep_type="conv"`) to preprocess the input images, and
+[`~models.perceiver.modeling_perceiver.PerceiverClassificationDecoder`] to decode the latent representation of
+[`PerceiverModel`] into classification logits.
""",
PERCEIVER_START_DOCSTRING,
)
@@ -1507,8 +1513,9 @@ class PerceiverForImageClassificationConvProcessing(PerceiverPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the image classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
Returns:
@@ -1586,11 +1593,10 @@ class PerceiverForImageClassificationConvProcessing(PerceiverPreTrainedModel):
@add_start_docstrings(
"""
-Example use of Perceiver for optical flow, for tasks such as Sintel and KITTI.
-[`PerceiverForOpticalFlow`] uses
-[`~models.perceiver.modeling_perceiver.PerceiverImagePreprocessor`] (with *prep_type="patches"*) to
-preprocess the input images, and [`~models.perceiver.modeling_perceiver.PerceiverOpticalFlowDecoder`]
-to decode the latent representation of [`PerceiverModel`].
+Example use of Perceiver for optical flow, for tasks such as Sintel and KITTI. [`PerceiverForOpticalFlow`] uses
+[`~models.perceiver.modeling_perceiver.PerceiverImagePreprocessor`] (with *prep_type="patches"*) to preprocess the
+input images, and [`~models.perceiver.modeling_perceiver.PerceiverOpticalFlowDecoder`] to decode the latent
+representation of [`PerceiverModel`].
As input, one concatenates 2 subsequent frames along the channel dimension and extract a 3 x 3 patch around each pixel
(leading to 3 x 3 x 3 x 2 = 54 values for each pixel). Fixed Fourier position encodings are used to encode the position
@@ -1713,27 +1719,26 @@ class PerceiverForOpticalFlow(PerceiverPreTrainedModel):
"""
Example use of Perceiver for multimodal (video) autoencoding, for tasks such as Kinetics-700.
-[`PerceiverForMultimodalAutoencoding`] uses
-[`~models.perceiver.modeling_perceiver.PerceiverMultimodalPreprocessor`] to preprocess the 3
-modalities: images, audio and class labels. This preprocessor uses modality-specific preprocessors to preprocess every
-modality separately, after which they are concatenated. Trainable position embeddings are used to pad each modality to
-the same number of channels to make concatenation along the time dimension possible. Next, one applies the Perceiver
-encoder.
+[`PerceiverForMultimodalAutoencoding`] uses [`~models.perceiver.modeling_perceiver.PerceiverMultimodalPreprocessor`] to
+preprocess the 3 modalities: images, audio and class labels. This preprocessor uses modality-specific preprocessors to
+preprocess every modality separately, after which they are concatenated. Trainable position embeddings are used to pad
+each modality to the same number of channels to make concatenation along the time dimension possible. Next, one applies
+the Perceiver encoder.
-[`~models.perceiver.modeling_perceiver.PerceiverMultimodalDecoder`] is used to decode the latent
-representation of [`PerceiverModel`]. This decoder uses each modality-specific decoder to construct
-queries. The decoder queries are created based on the inputs after preprocessing. However, autoencoding an entire video
-in a single forward pass is computationally infeasible, hence one only uses parts of the decoder queries to do
-cross-attention with the latent representation. This is determined by the subsampled indices for each modality, which
-can be provided as additional input to the forward pass of [`PerceiverForMultimodalAutoencoding`].
+[`~models.perceiver.modeling_perceiver.PerceiverMultimodalDecoder`] is used to decode the latent representation of
+[`PerceiverModel`]. This decoder uses each modality-specific decoder to construct queries. The decoder queries are
+created based on the inputs after preprocessing. However, autoencoding an entire video in a single forward pass is
+computationally infeasible, hence one only uses parts of the decoder queries to do cross-attention with the latent
+representation. This is determined by the subsampled indices for each modality, which can be provided as additional
+input to the forward pass of [`PerceiverForMultimodalAutoencoding`].
-[`~models.perceiver.modeling_perceiver.PerceiverMultimodalDecoder`] also pads the decoder queries of
-the different modalities to the same number of channels, in order to concatenate them along the time dimension. Next,
-cross-attention is performed with the latent representation of [`PerceiverModel`].
+[`~models.perceiver.modeling_perceiver.PerceiverMultimodalDecoder`] also pads the decoder queries of the different
+modalities to the same number of channels, in order to concatenate them along the time dimension. Next, cross-attention
+is performed with the latent representation of [`PerceiverModel`].
-Finally, [`~models.perceiver.modeling_perceiver.PerceiverMultiModalPostprocessor`] is used to turn
-this tensor into an actual video. It first splits up the output into the different modalities, and then applies the
-respective postprocessor for each modality.
+Finally, [`~models.perceiver.modeling_perceiver.PerceiverMultiModalPostprocessor`] is used to turn this tensor into an
+actual video. It first splits up the output into the different modalities, and then applies the respective
+postprocessor for each modality.
Note that, by masking the classification label during evaluation (i.e. simply providing a tensor of zeros for the
"label" modality), this auto-encoding model becomes a Kinetics 700 video classifier.
@@ -1871,8 +1876,9 @@ class PerceiverForMultimodalAutoencoding(PerceiverPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the image classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
Returns:
@@ -2576,11 +2582,11 @@ def generate_fourier_features(pos, num_bands, max_resolution=(224, 224), concat_
Whether to use a single phase (sin) or two (sin/cos) for each frequency band.
Returns:
- `torch.FloatTensor` of shape `(batch_size, sequence_length, n_channels)`: The Fourier position
- embeddings. If `concat_pos` is *True* and `sine_only` is *False*, output dimensions are ordered as:
- [dim_1, dim_2, ..., dim_d, sin(pi*f_1*dim_1), ..., sin(pi*f_K*dim_1), ..., sin(pi*f_1*dim_d), ...,
- sin(pi*f_K*dim_d), cos(pi*f_1*dim_1), ..., cos(pi*f_K*dim_1), ..., cos(pi*f_1*dim_d), ..., cos(pi*f_K*dim_d)],
- where dim_i is pos[:, i] and f_k is the kth frequency band.
+ `torch.FloatTensor` of shape `(batch_size, sequence_length, n_channels)`: The Fourier position embeddings. If
+ `concat_pos` is *True* and `sine_only` is *False*, output dimensions are ordered as: [dim_1, dim_2, ..., dim_d,
+ sin(pi*f_1*dim_1), ..., sin(pi*f_K*dim_1), ..., sin(pi*f_1*dim_d), ..., sin(pi*f_K*dim_d), cos(pi*f_1*dim_1),
+ ..., cos(pi*f_K*dim_1), ..., cos(pi*f_1*dim_d), ..., cos(pi*f_K*dim_d)], where dim_i is pos[:, i] and f_k is the
+ kth frequency band.
"""
batch_size = pos.shape[0]
diff --git a/src/transformers/models/perceiver/tokenization_perceiver.py b/src/transformers/models/perceiver/tokenization_perceiver.py
index 505109a6d3..644ec58ef7 100644
--- a/src/transformers/models/perceiver/tokenization_perceiver.py
+++ b/src/transformers/models/perceiver/tokenization_perceiver.py
@@ -28,8 +28,8 @@ class PerceiverTokenizer(PreTrainedTokenizer):
"""
Construct a Perceiver tokenizer. The Perceiver simply uses raw bytes utf-8 encoding.
- This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods.
- Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
+ this superclass for more information regarding those methods.
Args:
pad_token (`str`, *optional*, defaults to `"[PAD]"`):
@@ -41,8 +41,8 @@ class PerceiverTokenizer(PreTrainedTokenizer):
- When building a sequence using special tokens, this is not the token that is used for the end of
- sequence. The token used is the `sep_token`.
+ When building a sequence using special tokens, this is not the token that is used for the end of sequence.
+ The token used is the `sep_token`.
diff --git a/src/transformers/models/phobert/tokenization_phobert.py b/src/transformers/models/phobert/tokenization_phobert.py
index 826db73374..a37a5645ae 100644
--- a/src/transformers/models/phobert/tokenization_phobert.py
+++ b/src/transformers/models/phobert/tokenization_phobert.py
@@ -13,7 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" Tokenization classes for PhoBERT """
+""" Tokenization classes for PhoBERT"""
import os
@@ -69,8 +69,8 @@ class PhobertTokenizer(PreTrainedTokenizer):
"""
Construct a PhoBERT tokenizer. Based on Byte-Pair-Encoding.
- This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods.
- Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
+ this superclass for more information regarding those methods.
Args:
vocab_file (`str`):
@@ -92,8 +92,8 @@ class PhobertTokenizer(PreTrainedTokenizer):
- When building a sequence using special tokens, this is not the token that is used for the end of
- sequence. The token used is the `sep_token`.
+ When building a sequence using special tokens, this is not the token that is used for the end of sequence.
+ The token used is the `sep_token`.
diff --git a/src/transformers/models/prophetnet/configuration_prophetnet.py b/src/transformers/models/prophetnet/configuration_prophetnet.py
index 2ed8571e4b..a846be6298 100644
--- a/src/transformers/models/prophetnet/configuration_prophetnet.py
+++ b/src/transformers/models/prophetnet/configuration_prophetnet.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" ProphetNet model configuration """
+""" ProphetNet model configuration"""
from ...configuration_utils import PretrainedConfig
@@ -28,18 +28,18 @@ PROPHETNET_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class ProphetNetConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`ProphetNetModel`]. It is used
- to instantiate a ProphetNet model according to the specified arguments, defining the model architecture.
+ This is the configuration class to store the configuration of a [`ProphetNetModel`]. It is used to instantiate a
+ ProphetNet model according to the specified arguments, defining the model architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
activation_dropout (`float`, *optional*, defaults to 0.1):
The dropout ratio for activations inside the fully connected layer.
activation_function (`str` or `function`, *optional*, defaults to `"gelu"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"silu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"silu"` and `"gelu_new"` are supported.
vocab_size (`int`, *optional*, defaults to 30522):
Vocabulary size of the ProphetNET model. Defines the number of different tokens that can be represented by
the `inputs_ids` passed when calling [`ProphetNetModel`].
diff --git a/src/transformers/models/prophetnet/modeling_prophetnet.py b/src/transformers/models/prophetnet/modeling_prophetnet.py
index 8960a3deb9..5cf97de072 100644
--- a/src/transformers/models/prophetnet/modeling_prophetnet.py
+++ b/src/transformers/models/prophetnet/modeling_prophetnet.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" PyTorch ProphetNet model, ported from ProphetNet repo(fairsequery_states version). """
+""" PyTorch ProphetNet model, ported from ProphetNet repo(fairsequery_states version)."""
import copy
import math
@@ -51,9 +51,9 @@ PROPHETNET_PRETRAINED_MODEL_ARCHIVE_LIST = [
PROPHETNET_START_DOCSTRING = r"""
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
Original ProphetNet code can be found [here](https://github.com/microsoft/ProphetNet). Checkpoints were converted
from original Fairseq checkpoints. For more information on the checkpoint conversion, please take a look at the
@@ -66,8 +66,7 @@ PROPHETNET_START_DOCSTRING = r"""
Parameters:
config ([`ProphetNetConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
PROPHETNET_INPUTS_DOCSTRING = r"""
@@ -76,9 +75,8 @@ PROPHETNET_INPUTS_DOCSTRING = r"""
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
it.
- Indices can be obtained using [`ProphetNetTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`ProphetNetTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -91,9 +89,8 @@ PROPHETNET_INPUTS_DOCSTRING = r"""
decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
Indices of decoder input sequence tokens in the vocabulary.
- Indices can be obtained using [`ProphetNetTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`ProphetNetTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are decoder input IDs?](../glossary#decoder-input-ids)
@@ -102,8 +99,8 @@ PROPHETNET_INPUTS_DOCSTRING = r"""
`past_key_values`).
decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
- Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will
- also be used by default.
+ Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
+ be used by default.
head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*):
Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`:
@@ -123,19 +120,18 @@ PROPHETNET_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*):
- Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*:
- `attentions`) `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`,
- *optional*) is a sequence of hidden-states at the output of the last layer of the encoder. Used in the
- cross-attention of the decoder.
+ Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`)
+ `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of
+ hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder.
past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
Contains precomputed key and value hidden-states of the attention blocks. Can be used to speed up decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ `decoder_input_ids` of shape `(batch_size, sequence_length)`.
use_cache (`bool`, *optional*):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`).
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`).
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -152,9 +148,8 @@ PROPHETNET_STANDALONE_INPUTS_DOCSTRING = r"""
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
it.
- Indices can be obtained using [`ProphetNetTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`ProphetNetTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -268,45 +263,50 @@ class ProphetNetSeq2SeqLMOutput(ModelOutput):
Prediction scores of the predict stream language modeling head (scores for each vocabulary token before
SoftMax).
past_key_values (`List[torch.FloatTensor]`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- List of `torch.FloatTensor` of length `config.n_layers`, with each tensor of shape `(2, batch_size, num_attn_heads, decoder_sequence_length, embed_size_per_head)`).
+ List of `torch.FloatTensor` of length `config.n_layers`, with each tensor of shape `(2, batch_size,
+ num_attn_heads, decoder_sequence_length, embed_size_per_head)`).
Contains pre-computed hidden-states (key and values in the attention blocks) of the decoder that can be
used (see `past_key_values` input) to speed up sequential decoding.
decoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, decoder_sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, decoder_sequence_length, hidden_size)`.
Hidden-states of main stream of the decoder at the output of each layer plus the initial embedding outputs.
decoder_ngram_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, ngram * decoder_sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, ngram * decoder_sequence_length, hidden_size)`.
Hidden-states of the predict stream of the decoder at the output of each layer plus the initial embedding
outputs.
decoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_attn_heads, decoder_sequence_length, decoder_sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_attn_heads,
+ decoder_sequence_length, decoder_sequence_length)`.
Attentions weights of the decoder, after the attention softmax, used to compute the weighted average in the
self-attention heads.
decoder_ngram_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_attn_heads, decoder_sequence_length, decoder_sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_attn_heads,
+ decoder_sequence_length, decoder_sequence_length)`.
Attentions weights of the predict stream of the decoder, after the attention softmax, used to compute the
weighted average in the self-attention heads.
cross_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_attn_heads, encoder_sequence_length, decoder_sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_attn_heads,
+ encoder_sequence_length, decoder_sequence_length)`.
Attentions weights of the cross-attention layer of the decoder, after the attention softmax, used to
compute the weighted average in the
encoder_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*):
Sequence of hidden-states at the output of the last layer of the encoder of the model.
encoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, encoder_sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, encoder_sequence_length, hidden_size)`.
Hidden-states of the encoder at the output of each layer plus the initial embedding outputs.
encoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_attn_heads, encoder_sequence_length, encoder_sequence_length)`. Attentions weights of the encoder, after the attention
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_attn_heads,
+ encoder_sequence_length, encoder_sequence_length)`. Attentions weights of the encoder, after the attention
softmax, used to compute the weighted average in the self-attention heads.
"""
@@ -342,49 +342,55 @@ class ProphetNetSeq2SeqModelOutput(ModelOutput):
last_hidden_state (`torch.FloatTensor` of shape `(batch_size, decoder_sequence_length, hidden_size)`):
Sequence of main stream hidden-states at the output of the last layer of the decoder of the model.
- If `past_key_values` is used only the last hidden-state of the sequences of shape `(batch_size, 1, hidden_size)` is output.
+ If `past_key_values` is used only the last hidden-state of the sequences of shape `(batch_size, 1,
+ hidden_size)` is output.
last_hidden_state_ngram (`torch.FloatTensor` of shape `(batch_size,ngram * decoder_sequence_length, config.vocab_size)`):
Sequence of predict stream hidden-states at the output of the last layer of the decoder of the model.
past_key_values (`List[torch.FloatTensor]`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- List of `torch.FloatTensor` of length `config.n_layers`, with each tensor of shape `(2, batch_size, num_attn_heads, decoder_sequence_length, embed_size_per_head)`).
+ List of `torch.FloatTensor` of length `config.n_layers`, with each tensor of shape `(2, batch_size,
+ num_attn_heads, decoder_sequence_length, embed_size_per_head)`).
Contains pre-computed hidden-states (key and values in the attention blocks) of the decoder that can be
used (see `past_key_values` input) to speed up sequential decoding.
decoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, decoder_sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, decoder_sequence_length, hidden_size)`.
Hidden-states of main stream of the decoder at the output of each layer plus the initial embedding outputs.
decoder_ngram_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, ngram * decoder_sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, ngram * decoder_sequence_length, hidden_size)`.
Hidden-states of the predict stream of the decoder at the output of each layer plus the initial embedding
outputs.
decoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_attn_heads, decoder_sequence_length, decoder_sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_attn_heads,
+ decoder_sequence_length, decoder_sequence_length)`.
Attentions weights of the decoder, after the attention softmax, used to compute the weighted average in the
self-attention heads.
decoder_ngram_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_attn_heads, decoder_sequence_length, decoder_sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_attn_heads,
+ decoder_sequence_length, decoder_sequence_length)`.
Attentions weights of the predict stream of the decoder, after the attention softmax, used to compute the
weighted average in the
cross_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_attn_heads, encoder_sequence_length, decoder_sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_attn_heads,
+ encoder_sequence_length, decoder_sequence_length)`.
Attentions weights of the cross-attention layer of the decoder, after the attention softmax, used to
compute the weighted average in the
encoder_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*):
Sequence of hidden-states at the output of the last layer of the encoder of the model.
encoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, encoder_sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, encoder_sequence_length, hidden_size)`.
Hidden-states of the encoder at the output of each layer plus the initial embedding outputs.
encoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_attn_heads, encoder_sequence_length, encoder_sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_attn_heads,
+ encoder_sequence_length, encoder_sequence_length)`.
Attentions weights of the encoder, after the attention softmax, used to compute the weighted average in the
self-attention heads.
@@ -420,37 +426,42 @@ class ProphetNetDecoderModelOutput(ModelOutput):
last_hidden_state (`torch.FloatTensor` of shape `(batch_size, decoder_sequence_length, hidden_size)`):
Sequence of main stream hidden-states at the output of the last layer of the decoder of the model.
- If `past_key_values` is used only the last hidden-state of the sequences of shape `(batch_size, 1, hidden_size)` is output.
+ If `past_key_values` is used only the last hidden-state of the sequences of shape `(batch_size, 1,
+ hidden_size)` is output.
last_hidden_state_ngram (`torch.FloatTensor` of shape `(batch_size, ngram * decoder_sequence_length, config.vocab_size)`):
Sequence of predict stream hidden-states at the output of the last layer of the decoder of the model.
past_key_values (`List[torch.FloatTensor]`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- List of `torch.FloatTensor` of length `config.n_layers`, with each tensor of shape `(2, batch_size, num_attn_heads, decoder_sequence_length, embed_size_per_head)`).
+ List of `torch.FloatTensor` of length `config.n_layers`, with each tensor of shape `(2, batch_size,
+ num_attn_heads, decoder_sequence_length, embed_size_per_head)`).
Contains pre-computed hidden-states (key and values in the attention blocks) of the decoder that can be
used (see `past_key_values` input) to speed up sequential decoding.
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, decoder_sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, decoder_sequence_length, hidden_size)`.
Hidden-states of main stream of the decoder at the output of each layer plus the initial embedding outputs.
ngram_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, ngram * decoder_sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, ngram * decoder_sequence_length, hidden_size)`.
Hidden-states of the predict stream of the decoder at the output of each layer plus the initial embedding
outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_attn_heads, decoder_sequence_length, decoder_sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_attn_heads,
+ decoder_sequence_length, decoder_sequence_length)`.
Attentions weights of the decoder, after the attention softmax, used to compute the weighted average in the
self-attention heads.
ngram_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_attn_heads, decoder_sequence_length, decoder_sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_attn_heads,
+ decoder_sequence_length, decoder_sequence_length)`.
Attentions weights of the predict stream of the decoder, after the attention softmax, used to compute the
weighted average in the
cross_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_attn_heads, encoder_sequence_length, decoder_sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_attn_heads,
+ encoder_sequence_length, decoder_sequence_length)`.
Attentions weights of the cross-attention layer of the decoder, after the attention softmax, used to
compute the weighted average in the
@@ -481,33 +492,37 @@ class ProphetNetDecoderLMOutput(ModelOutput):
Prediction scores of the predict stream language modeling head (scores for each vocabulary token before
SoftMax).
past_key_values (`List[torch.FloatTensor]`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- List of `torch.FloatTensor` of length `config.n_layers`, with each tensor of shape `(2, batch_size, num_attn_heads, decoder_sequence_length, embed_size_per_head)`).
+ List of `torch.FloatTensor` of length `config.n_layers`, with each tensor of shape `(2, batch_size,
+ num_attn_heads, decoder_sequence_length, embed_size_per_head)`).
Contains pre-computed hidden-states (key and values in the attention blocks) of the decoder that can be
used (see `past_key_values` input) to speed up sequential decoding.
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, decoder_sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, decoder_sequence_length, hidden_size)`.
Hidden-states of main stream of the decoder at the output of each layer plus the initial embedding outputs.
ngram_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, ngram * decoder_sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, ngram * decoder_sequence_length, hidden_size)`.
Hidden-states of the predict stream of the decoder at the output of each layer plus the initial embedding
outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_attn_heads, decoder_sequence_length, decoder_sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_attn_heads,
+ decoder_sequence_length, decoder_sequence_length)`.
Attentions weights of the decoder, after the attention softmax, used to compute the weighted average in the
self-attention heads.
ngram_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_attn_heads, decoder_sequence_length, decoder_sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_attn_heads,
+ decoder_sequence_length, decoder_sequence_length)`.
Attentions weights of the predict stream of the decoder, after the attention softmax, used to compute the
weighted average in the
cross_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_attn_heads, encoder_sequence_length, decoder_sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_attn_heads,
+ encoder_sequence_length, decoder_sequence_length)`.
Attentions weights of the cross-attention layer of the decoder, after the attention softmax, used to
compute the weighted average in the
@@ -1229,8 +1244,8 @@ class ProphetNetDecoderLayer(nn.Module):
class ProphetNetEncoder(ProphetNetPreTrainedModel):
r"""
word_embeddings (`torch.nn.Embeddings` of shape `(config.vocab_size, config.hidden_size)`, *optional*):
- The word embedding parameters. This can be used to initialize [`ProphetNetEncoder`] with
- pre-defined word embeddings instead of randomly initialized word embeddings.
+ The word embedding parameters. This can be used to initialize [`ProphetNetEncoder`] with pre-defined word
+ embeddings instead of randomly initialized word embeddings.
"""
def __init__(self, config: ProphetNetConfig, word_embeddings: nn.Embedding = None):
@@ -1369,8 +1384,8 @@ class ProphetNetEncoder(ProphetNetPreTrainedModel):
class ProphetNetDecoder(ProphetNetPreTrainedModel):
r"""
word_embeddings (`torch.nn.Embeddings` of shape `(config.vocab_size, config.hidden_size)`, *optional*):
- The word embedding parameters. This can be used to initialize [`ProphetNetEncoder`] with
- pre-defined word embeddings instead of randomly initialized word embeddings.
+ The word embedding parameters. This can be used to initialize [`ProphetNetEncoder`] with pre-defined word
+ embeddings instead of randomly initialized word embeddings.
"""
def __init__(self, config: ProphetNetConfig, word_embeddings: nn.Embedding = None):
@@ -1436,12 +1451,12 @@ class ProphetNetDecoder(ProphetNetPreTrainedModel):
past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
Contains precomputed key and value hidden-states of the attention blocks. Can be used to speed up decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ `decoder_input_ids` of shape `(batch_size, sequence_length)`.
use_cache (`bool`, *optional*):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`).
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`).
- 1 for tokens that are **not masked**,
- 0 for tokens that are **masked**.
@@ -1903,7 +1918,8 @@ class ProphetNetForConditionalGeneration(ProphetNetPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[-100, 0, ..., config.vocab_size - 1]`. All labels set to `-100` are ignored (masked), the loss is only computed for
+ Labels for computing the sequence classification/regression loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size - 1]`. All labels set to `-100` are ignored (masked), the loss is only computed for
labels in `[0, ..., config.vocab_size]`
Returns:
@@ -2134,12 +2150,12 @@ class ProphetNetForCausalLM(ProphetNetPreTrainedModel):
past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
Contains precomputed key and value hidden-states of the attention blocks. Can be used to speed up decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ `decoder_input_ids` of shape `(batch_size, sequence_length)`.
use_cache (`bool`, *optional*):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`).
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`).
- 1 for tokens that are **not masked**,
- 0 for tokens that are **masked**.
@@ -2293,8 +2309,8 @@ class ProphetNetForCausalLM(ProphetNetPreTrainedModel):
class ProphetNetDecoderWrapper(ProphetNetPreTrainedModel):
"""
- This is a wrapper class, so that [`ProphetNetForCausalLM`] can correctly be loaded from
- pretrained prophetnet classes.
+ This is a wrapper class, so that [`ProphetNetForCausalLM`] can correctly be loaded from pretrained prophetnet
+ classes.
"""
def __init__(self, config):
diff --git a/src/transformers/models/prophetnet/tokenization_prophetnet.py b/src/transformers/models/prophetnet/tokenization_prophetnet.py
index 9866b7fd38..6ee7f18198 100644
--- a/src/transformers/models/prophetnet/tokenization_prophetnet.py
+++ b/src/transformers/models/prophetnet/tokenization_prophetnet.py
@@ -56,8 +56,8 @@ class ProphetNetTokenizer(PreTrainedTokenizer):
r"""
Construct a ProphetNetTokenizer. Based on WordPiece.
- This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods.
- Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
+ this superclass for more information regarding those methods.
Args:
vocab_file (`str`):
@@ -77,9 +77,8 @@ class ProphetNetTokenizer(PreTrainedTokenizer):
sequence classification or for a text and a question for question answering. It is also used as the last
token of a sequence built with special tokens.
x_sep_token (`str`, *optional*, defaults to `"[X_SEP]"`):
- Special second separator token, which can be generated by
- [`ProphetNetForConditionalGeneration`]. It is used to separate bullet-point like
- sentences in summarization, *e.g.*.
+ Special second separator token, which can be generated by [`ProphetNetForConditionalGeneration`]. It is
+ used to separate bullet-point like sentences in summarization, *e.g.*.
pad_token (`str`, *optional*, defaults to `"[PAD]"`):
The token used for padding, for example when batching sequences of different lengths.
cls_token (`str`, *optional*, defaults to `"[CLS]"`):
@@ -91,7 +90,8 @@ class ProphetNetTokenizer(PreTrainedTokenizer):
tokenize_chinese_chars (`bool`, *optional*, defaults to `True`):
Whether or not to tokenize Chinese characters.
- This should likely be deactivated for Japanese (see this [issue](https://github.com/huggingface/transformers/issues/328)).
+ This should likely be deactivated for Japanese (see this
+ [issue](https://github.com/huggingface/transformers/issues/328)).
strip_accents: (`bool`, *optional*):
Whether or not to strip all accents. If this option is not specified, then it will be determined by the
value for `lowercase` (as in the original BERT).
@@ -231,8 +231,7 @@ class ProphetNetTokenizer(PreTrainedTokenizer):
Optional second list of IDs for sequence pairs.
Returns:
- `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given
- sequence(s).
+ `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
"""
sep = [self.sep_token_id]
if token_ids_1 is None:
diff --git a/src/transformers/models/qdqbert/configuration_qdqbert.py b/src/transformers/models/qdqbert/configuration_qdqbert.py
index a30353a3b5..b6ac980eb5 100644
--- a/src/transformers/models/qdqbert/configuration_qdqbert.py
+++ b/src/transformers/models/qdqbert/configuration_qdqbert.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" QDQBERT model configuration """
+""" QDQBERT model configuration"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging
@@ -28,12 +28,13 @@ QDQBERT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class QDQBertConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`QDQBertModel`]. It is used to
- instantiate an QDQBERT model according to the specified arguments, defining the model architecture. Instantiating a
- configuration with the defaults will yield a similar configuration to that of the BERT [bert-base-uncased](https://huggingface.co/bert-base-uncased) architecture.
+ This is the configuration class to store the configuration of a [`QDQBertModel`]. It is used to instantiate an
+ QDQBERT model according to the specified arguments, defining the model architecture. Instantiating a configuration
+ with the defaults will yield a similar configuration to that of the BERT
+ [bert-base-uncased](https://huggingface.co/bert-base-uncased) architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
@@ -49,8 +50,8 @@ class QDQBertConfig(PretrainedConfig):
intermediate_size (`int`, *optional*, defaults to 3072):
Dimension of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"selu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"selu"` and `"gelu_new"` are supported.
hidden_dropout_prob (`float`, *optional*, defaults to 0.1):
The dropout probabilitiy for all fully connected layers in the embeddings, encoder, and pooler.
attention_probs_dropout_prob (`float`, *optional*, defaults to 0.1):
diff --git a/src/transformers/models/qdqbert/modeling_qdqbert.py b/src/transformers/models/qdqbert/modeling_qdqbert.py
index 0d6495c69a..fd1c363fdb 100755
--- a/src/transformers/models/qdqbert/modeling_qdqbert.py
+++ b/src/transformers/models/qdqbert/modeling_qdqbert.py
@@ -13,7 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" PyTorch QDQBERT model. """
+""" PyTorch QDQBERT model."""
import math
@@ -760,19 +760,18 @@ class QDQBertPreTrainedModel(PreTrainedModel):
QDQBERT_START_DOCSTRING = r"""
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
Parameters:
config ([`QDQBertConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
QDQBERT_INPUTS_DOCSTRING = r"""
@@ -780,9 +779,8 @@ QDQBERT_INPUTS_DOCSTRING = r"""
input_ids (`torch.LongTensor` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`BertTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`BertTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
@@ -793,14 +791,16 @@ QDQBERT_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
@@ -810,9 +810,9 @@ QDQBERT_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -836,10 +836,9 @@ class QDQBertModel(QDQBertPreTrainedModel):
all you need](https://arxiv.org/abs/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit,
Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin.
- To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration
- set to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder`
- argument and `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an
- input to the forward pass.
+ To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set
+ to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and
+ `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass.
"""
def __init__(self, config, add_pooling_layer=True):
@@ -905,12 +904,12 @@ class QDQBertModel(QDQBertPreTrainedModel):
past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ `decoder_input_ids` of shape `(batch_size, sequence_length)`.
use_cache (`bool`, *optional*):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`).
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`).
"""
output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
output_hidden_states = (
@@ -1008,7 +1007,7 @@ class QDQBertModel(QDQBertPreTrainedModel):
@add_start_docstrings(
- """QDQBERT Model with a `language modeling` head on top for CLM fine-tuning. """, QDQBERT_START_DOCSTRING
+ """QDQBERT Model with a `language modeling` head on top for CLM fine-tuning.""", QDQBERT_START_DOCSTRING
)
class QDQBertLMHeadModel(QDQBertPreTrainedModel):
@@ -1069,12 +1068,12 @@ class QDQBertLMHeadModel(QDQBertPreTrainedModel):
past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ `decoder_input_ids` of shape `(batch_size, sequence_length)`.
use_cache (`bool`, *optional*):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`).
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`).
Returns:
@@ -1157,7 +1156,7 @@ class QDQBertLMHeadModel(QDQBertPreTrainedModel):
return reordered_past
-@add_start_docstrings("""QDQBERT Model with a `language modeling` head on top. """, QDQBERT_START_DOCSTRING)
+@add_start_docstrings("""QDQBERT Model with a `language modeling` head on top.""", QDQBERT_START_DOCSTRING)
class QDQBertForMaskedLM(QDQBertPreTrainedModel):
_keys_to_ignore_on_load_unexpected = [r"pooler"]
@@ -1208,8 +1207,9 @@ class QDQBertForMaskedLM(QDQBertPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1265,7 +1265,7 @@ class QDQBertForMaskedLM(QDQBertPreTrainedModel):
@add_start_docstrings(
- """Bert Model with a `next sentence prediction (classification)` head on top. """,
+ """Bert Model with a `next sentence prediction (classification)` head on top.""",
QDQBERT_START_DOCSTRING,
)
class QDQBertForNextSentencePrediction(QDQBertPreTrainedModel):
@@ -1405,8 +1405,9 @@ class QDQBertForSequenceClassification(QDQBertPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1501,7 +1502,8 @@ class QDQBertForMultipleChoice(QDQBertPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
+ Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
+ num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
`input_ids` above)
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1688,12 +1690,12 @@ class QDQBertForQuestionAnswering(QDQBertPreTrainedModel):
r"""
start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the start of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the end of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
diff --git a/src/transformers/models/rag/configuration_rag.py b/src/transformers/models/rag/configuration_rag.py
index 978b01cdd2..f009c8af8c 100644
--- a/src/transformers/models/rag/configuration_rag.py
+++ b/src/transformers/models/rag/configuration_rag.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" RAG model configuration """
+""" RAG model configuration"""
import copy
@@ -21,14 +21,12 @@ from ...file_utils import add_start_docstrings
RAG_CONFIG_DOC = r"""
- [`RagConfig`] stores the configuration of a *RagModel*. Configuration objects inherit from
- [`PretrainedConfig`] and can be used to control the model outputs. Read the documentation from
- [`PretrainedConfig`] for more information.
+ [`RagConfig`] stores the configuration of a *RagModel*. Configuration objects inherit from [`PretrainedConfig`] and
+ can be used to control the model outputs. Read the documentation from [`PretrainedConfig`] for more information.
Args:
title_sep (`str`, *optional*, defaults to `" / "`):
- Separator inserted between the title and the text of the retrieved document when calling
- [`RagRetriever`].
+ Separator inserted between the title and the text of the retrieved document when calling [`RagRetriever`].
doc_sep (`str`, *optional*, defaults to `" // "`):
Separator inserted between the the text of the retrieved document and the original input when calling
[`RagRetriever`].
@@ -47,8 +45,8 @@ RAG_CONFIG_DOC = r"""
dataset_split (`str`, *optional*, defaults to `"train"`)
Which split of the `dataset` to load.
index_name (`str`, *optional*, defaults to `"compressed"`)
- The index name of the index associated with the `dataset`. One can choose between `"legacy"`,
- `"exact"` and `"compressed"`.
+ The index name of the index associated with the `dataset`. One can choose between `"legacy"`, `"exact"` and
+ `"compressed"`.
index_path (`str`, *optional*)
The path to the serialized faiss index on disk.
passages_path: (`str`, *optional*):
@@ -57,8 +55,8 @@ RAG_CONFIG_DOC = r"""
use_dummy_dataset (`bool`, *optional*, defaults to `False`)
Whether to load a "dummy" variant of the dataset specified by `dataset`.
label_smoothing (`float`, *optional*, defaults to 0.0):
- Only relevant if `return_loss` is set to `True`. Controls the `epsilon` parameter value for label
- smoothing in the loss calculation. If set to 0, no label smoothing is performed.
+ Only relevant if `return_loss` is set to `True`. Controls the `epsilon` parameter value for label smoothing
+ in the loss calculation. If set to 0, no label smoothing is performed.
do_marginalize (`bool`, *optional*, defaults to `False`):
If `True`, the logits are marginalized over all documents by making use of
`torch.nn.functional.log_softmax`.
@@ -174,8 +172,8 @@ class RagConfig(PretrainedConfig):
cls, question_encoder_config: PretrainedConfig, generator_config: PretrainedConfig, **kwargs
) -> PretrainedConfig:
r"""
- Instantiate a [`EncoderDecoderConfig`] (or a derived class) from a pre-trained encoder model
- configuration and decoder model configuration.
+ Instantiate a [`EncoderDecoderConfig`] (or a derived class) from a pre-trained encoder model configuration and
+ decoder model configuration.
Returns:
[`EncoderDecoderConfig`]: An instance of a configuration object
@@ -184,8 +182,7 @@ class RagConfig(PretrainedConfig):
def to_dict(self):
"""
- Serializes this instance to a Python dictionary. Override the default
- [`~PretrainedConfig.to_dict`].
+ Serializes this instance to a Python dictionary. Override the default [`~PretrainedConfig.to_dict`].
Returns:
`Dict[str, any]`: Dictionary of all the attributes that make up this configuration instance,
diff --git a/src/transformers/models/rag/modeling_rag.py b/src/transformers/models/rag/modeling_rag.py
index 1f3f0d41d4..084601e920 100644
--- a/src/transformers/models/rag/modeling_rag.py
+++ b/src/transformers/models/rag/modeling_rag.py
@@ -52,13 +52,14 @@ class RetrievAugLMMarginOutput(ModelOutput):
Score between each retrieved document embeddings (see `retrieved_doc_embeds`) and
`question_encoder_last_hidden_state`.
past_key_values (`List[torch.FloatTensor]`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- List of `torch.FloatTensor` of length `config.n_layers`, with each tensor of shape `(2, batch_size, num_heads, sequence_length, embed_size_per_head)`).
+ List of `torch.FloatTensor` of length `config.n_layers`, with each tensor of shape `(2, batch_size,
+ num_heads, sequence_length, embed_size_per_head)`).
Contains precomputed hidden-states (key and values in the attention blocks) of the decoder that can be used
(see `past_key_values` input) to speed up sequential decoding.
retrieved_doc_embeds (`torch.FloatTensor` of shape `(batch_size, config.n_docs, hidden_size)`, *optional*, returned when *output_retrieved=True*):
- Embedded documents retrieved by the retriever. Is used with `question_encoder_last_hidden_state` to
- compute the `doc_scores`.
+ Embedded documents retrieved by the retriever. Is used with `question_encoder_last_hidden_state` to compute
+ the `doc_scores`.
retrieved_doc_ids (`torch.LongTensor` of shape `(batch_size, config.n_docs)`, *optional*, returned when *output_retrieved=True*):
The indexes of the embedded documents retrieved by the retriever.
context_input_ids (`torch.LongTensor` of shape `(batch_size * config.n_docs, config.max_combined_length)`, *optional*, returned when *output_retrieved=True*):
@@ -70,39 +71,43 @@ class RetrievAugLMMarginOutput(ModelOutput):
Sequence of hidden states at the output of the last layer of the question encoder pooled output of the
model.
question_enc_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings and one for the output of each
- layer) of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings and one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden states of the question encoder at the output of each layer plus the initial embedding outputs.
question_enc_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the question encoder, after the attention softmax, used to compute the weighted
average in the self-attention heads.
generator_enc_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
Sequence of hidden-states at the output of the last layer of the generator encoder of the model.
generator_enc_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings and one for the output of each
- layer) of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings and one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden states of the generator encoder at the output of each layer plus the initial embedding outputs.
generator_enc_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the generator encoder, after the attention softmax, used to compute the weighted
average in the self-attention heads.
generator_dec_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings and one for the output of each
- layer) of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings and one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden states of the generator decoder at the output of each layer plus the initial embedding outputs.
generator_dec_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the generator decoder, after the attention softmax, used to compute the weighted
average in the self-attention heads.
generator_cross_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Cross-attentions weights of the generator decoder, after the attention softmax, used to compute the
weighted average in the cross-attention heads.
@@ -138,13 +143,14 @@ class RetrievAugLMOutput(ModelOutput):
Score between each retrieved document embeddings (see `retrieved_doc_embeds`) and
`question_encoder_last_hidden_state`.
past_key_values (`List[torch.FloatTensor]`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- List of `torch.FloatTensor` of length `config.n_layers`, with each tensor of shape `(2, batch_size, num_heads, sequence_length, embed_size_per_head)`).
+ List of `torch.FloatTensor` of length `config.n_layers`, with each tensor of shape `(2, batch_size,
+ num_heads, sequence_length, embed_size_per_head)`).
Contains precomputed hidden-states (key and values in the attention blocks) of the decoder that can be used
(see `past_key_values` input) to speed up sequential decoding.
retrieved_doc_embeds (`torch.FloatTensor` of shape `(batch_size, config.n_docs, hidden_size)`, *optional*, returned when *output_retrieved=True*):
- Embedded documents retrieved by the retriever. Is used with `question_encoder_last_hidden_state` to
- compute the `doc_scores`.
+ Embedded documents retrieved by the retriever. Is used with `question_encoder_last_hidden_state` to compute
+ the `doc_scores`.
retrieved_doc_ids (`torch.LongTensor` of shape `(batch_size, config.n_docs)`, *optional*, returned when *output_retrieved=True*):
The indexes of the embedded documents retrieved by the retriever.
context_input_ids (`torch.LongTensor` of shape `(batch_size * config.n_docs, config.max_combined_length)`, *optional*, returned when *output_retrieved=True*):
@@ -156,39 +162,43 @@ class RetrievAugLMOutput(ModelOutput):
Sequence of hidden states at the output of the last layer of the question encoder pooled output of the
model.
question_enc_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings and one for the output of each
- layer) of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings and one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden states of the question encoder at the output of each layer plus the initial embedding outputs.
question_enc_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the question encoder, after the attention softmax, used to compute the weighted
average in the self-attention heads.
generator_enc_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
Sequence of hidden-states at the output of the last layer of the generator encoder of the model.
generator_enc_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings and one for the output of each
- layer) of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings and one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden states of the generator encoder at the output of each layer plus the initial embedding outputs.
generator_enc_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the generator encoder, after the attention softmax, used to compute the weighted
average in the self-attention heads.
generator_dec_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings and one for the output of each
- layer) of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings and one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden states of the generator decoder at the output of each layer plus the initial embedding outputs.
generator_dec_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the generator decoder, after the attention softmax, used to compute the weighted
average in the self-attention heads.
generator_cross_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Cross-attentions weights of the generator decoder, after the attention softmax, used to compute the
weighted average in the cross-attention heads.
@@ -244,35 +254,35 @@ class RagPreTrainedModel(PreTrainedModel):
Instantiates an question encoder and a generator from one or two base classes of the library from pretrained
model checkpoints.
- The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated). To
- train the model, you need to first set it back in training mode with `model.train()`.
+ The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated). To train
+ the model, you need to first set it back in training mode with `model.train()`.
Params:
question_encoder_pretrained_model_name_or_path (:obj: *str*, *optional*, defaults to *None*):
Information necessary to initiate the question encoder. Can be either:
- A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co.
- Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under
- a user or organization name, like `dbmdz/bert-base-german-cased`.
+ Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under a
+ user or organization name, like `dbmdz/bert-base-german-cased`.
- A path to a *directory* containing model weights saved using
[`~PreTrainedModel.save_pretrained`], e.g., `./my_model_directory/`.
- A path or url to a *tensorflow index checkpoint file* (e.g, `./tf_model/model.ckpt.index`). In
- this case, `from_tf` should be set to `True` and a configuration object should be provided
- as `config` argument. This loading path is slower than converting the TensorFlow checkpoint in
- a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards.
+ this case, `from_tf` should be set to `True` and a configuration object should be provided as
+ `config` argument. This loading path is slower than converting the TensorFlow checkpoint in a
+ PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards.
generator_pretrained_model_name_or_path (:obj: *str*, *optional*, defaults to *None*):
Information necessary to initiate the generator. Can be either:
- A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co.
- Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under
- a user or organization name, like `dbmdz/bert-base-german-cased`.
+ Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under a
+ user or organization name, like `dbmdz/bert-base-german-cased`.
- A path to a *directory* containing model weights saved using
[`~PreTrainedModel.save_pretrained`], e.g., `./my_model_directory/`.
- A path or url to a *tensorflow index checkpoint file* (e.g, `./tf_model/model.ckpt.index`). In
- this case, `from_tf` should be set to `True` and a configuration object should be provided
- as `config` argument. This loading path is slower than converting the TensorFlow checkpoint in
- a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards.
+ this case, `from_tf` should be set to `True` and a configuration object should be provided as
+ `config` argument. This loading path is slower than converting the TensorFlow checkpoint in a
+ PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards.
model_args (remaining positional arguments, *optional*):
All remaining positional arguments will be passed to the underlying model's `__init__` method.
@@ -379,23 +389,22 @@ RAG_START_DOCSTRING = r"""
pass, we encode the input with the question encoder and pass it to the retriever to extract relevant context
documents. The documents are then prepended to the input. Such contextualized inputs is passed to the generator.
- The question encoder can be any *autoencoding* model, preferably [`DPRQuestionEncoder`], and the
- generator can be any *seq2seq* model, preferably [`BartForConditionalGeneration`].
+ The question encoder can be any *autoencoding* model, preferably [`DPRQuestionEncoder`], and the generator can be
+ any *seq2seq* model, preferably [`BartForConditionalGeneration`].
- The model can be initialized with a [`RagRetriever`] for end-to-end generation or used in
- combination with the outputs of a retriever in multiple steps---see examples for more details. The model is
- compatible any *autoencoding* model as the `question_encoder` and any *seq2seq* model with language model head as
- the `generator`. It has been tested with [`DPRQuestionEncoder`] as the `question_encoder`
- and [`BartForConditionalGeneration`] or [`T5ForConditionalGeneration`] as the
- `generator`.
+ The model can be initialized with a [`RagRetriever`] for end-to-end generation or used in combination with the
+ outputs of a retriever in multiple steps---see examples for more details. The model is compatible any
+ *autoencoding* model as the `question_encoder` and any *seq2seq* model with language model head as the `generator`.
+ It has been tested with [`DPRQuestionEncoder`] as the `question_encoder` and [`BartForConditionalGeneration`] or
+ [`T5ForConditionalGeneration`] as the `generator`.
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
Args:
@@ -415,9 +424,9 @@ RAG_START_DOCSTRING = r"""
RAG_FORWARD_INPUTS_DOCSTRING = r"""
Args:
input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
- Indices of input sequence tokens in the vocabulary. [`RagConfig`], used to initialize
- the model, specifies which generator to use, it also specifies a compatible generator tokenizer. Use that
- tokenizer class to obtain the indices.
+ Indices of input sequence tokens in the vocabulary. [`RagConfig`], used to initialize the model, specifies
+ which generator to use, it also specifies a compatible generator tokenizer. Use that tokenizer class to
+ obtain the indices.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -429,41 +438,41 @@ RAG_FORWARD_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*)
Tuple consists of (`generator_enc_last_hidden_state`, *optional*: `generator_enc_hidden_states`,
- *optional*: `generator_enc_attentions`). `generator_enc_last_hidden_state` of shape
- `(batch_size, n_docs * sequence_length, hidden_size)` is a sequence of hidden-states at the output of
- the last layer of the generator's encoder.
+ *optional*: `generator_enc_attentions`). `generator_enc_last_hidden_state` of shape `(batch_size, n_docs *
+ sequence_length, hidden_size)` is a sequence of hidden-states at the output of the last layer of the
+ generator's encoder.
Used by the ([`RagModel`]) model during decoding.
decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
Provide for generation tasks. *None* by default, construct as per instructions for the generator model
you're using with your RAG instance.
decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
- Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will
- also be used by default.
+ Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
+ be used by default.
past_key_values (`tuple(tuple(torch.FloatTensor))`):
Tuple consists of two elements: `encoder_outputs` of the RAG model (see `encoder_outputs`) and
- `past_key_values` of the underlying generator. Can be used to speed up decoding.
- `past_key_values` are used in the ([`RagTokenForGeneration`]) model during
- decoding.
+ `past_key_values` of the underlying generator. Can be used to speed up decoding. `past_key_values` are used
+ in the ([`RagTokenForGeneration`]) model during decoding.
doc_scores (`torch.FloatTensor` of shape `(batch_size, config.n_docs)`):
Score between each retrieved document embeddings (see `retrieved_doc_embeds`) and
- `question_encoder_last_hidden_state`. If the model has is not initialized with a `retriever`
- `doc_scores` has to be provided to the forward pass. `doc_scores` can be computed via
- `question_encoder_last_hidden_state` and `retrieved_doc_embeds`, see examples for more
- information.
+ `question_encoder_last_hidden_state`. If the model has is not initialized with a `retriever` `doc_scores`
+ has to be provided to the forward pass. `doc_scores` can be computed via
+ `question_encoder_last_hidden_state` and `retrieved_doc_embeds`, see examples for more information.
context_input_ids (`torch.LongTensor` of shape `(batch_size * config.n_docs, config.max_combined_length)`, *optional*, returned when *output_retrieved=True*):
Input IDs post-processed from the retrieved documents and the question encoder `input_ids` by the
retriever.
- If the model has is not initialized with a `retriever` ``context_input_ids` has to be provided to the forward pass. `context_input_ids` are returned by [`~RagRetriever.__call__`]. context_attention_mask (`torch.LongTensor` of shape `(batch_size * config.n_docs, config.max_combined_length)`, *optional*, returned when *output_retrieved=True*): Attention mask post-processed from the retrieved documents and the question encoder `input_ids` by the
- retriever.
+ If the model has is not initialized with a `retriever` ``context_input_ids` has to be provided to the
+ forward pass. `context_input_ids` are returned by [`~RagRetriever.__call__`]. context_attention_mask
+ (`torch.LongTensor` of shape `(batch_size * config.n_docs, config.max_combined_length)`, *optional*,
+ returned when *output_retrieved=True*): Attention mask post-processed from the retrieved documents and the
+ question encoder `input_ids` by the retriever.
- If the model has is not initialized with a `retriever` `context_attention_mask` has to be provided
- to the forward pass. `context_attention_mask` are returned by
- [`~RagRetriever.__call__`].
+ If the model has is not initialized with a `retriever` `context_attention_mask` has to be provided to the
+ forward pass. `context_attention_mask` are returned by [`~RagRetriever.__call__`].
use_cache (`bool`, *optional*, defaults to `True`):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`).
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`).
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -471,8 +480,8 @@ RAG_FORWARD_INPUTS_DOCSTRING = r"""
Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
more detail.
output_retrieved(`bool`, *optional*):
- Whether or not to return the `retrieved_doc_embeds`, `retrieved_doc_ids`,
- `context_input_ids` and `context_attention_mask`. See returned tensors for more detail.
+ Whether or not to return the `retrieved_doc_embeds`, `retrieved_doc_ids`, `context_input_ids` and
+ `context_attention_mask`. See returned tensors for more detail.
n_docs (`int`, *optional*, defaults to `config.n_docs``)
Number of documents to retrieve and/or number of documents for which to generate an answer.
"""
@@ -775,11 +784,11 @@ class RagSequenceForGeneration(RagPreTrainedModel):
):
r"""
exclude_bos_score (`bool`, *optional*):
- Only relevant if `labels` is passed. If `True`, the score of the BOS token is disregarded when
- computing the loss.
+ Only relevant if `labels` is passed. If `True`, the score of the BOS token is disregarded when computing
+ the loss.
reduce_loss (`bool`, *optional*):
- Only relevant if `labels` is passed. If `True`, the NLL loss is reduced using the
- `torch.Tensor.sum` operation.
+ Only relevant if `labels` is passed. If `True`, the NLL loss is reduced using the `torch.Tensor.sum`
+ operation.
kwargs (`Dict[str, any]`, optional, defaults to *{}*):
Legacy dictionary, which is required so that model can use *generate()* function.
@@ -898,9 +907,8 @@ class RagSequenceForGeneration(RagPreTrainedModel):
**model_kwargs
):
"""
- Implements RAG sequence "thorough" decoding. Read the
- [`~generation_utils.GenerationMixin.generate`]` documentation for more information on how to
- set other generate input parameters.
+ Implements RAG sequence "thorough" decoding. Read the [`~generation_utils.GenerationMixin.generate`]`
+ documentation for more information on how to set other generate input parameters.
Args:
input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -917,27 +925,25 @@ class RagSequenceForGeneration(RagPreTrainedModel):
Input IDs post-processed from the retrieved documents and the question encoder input_ids by the
retriever.
context_attention_mask (`torch.LongTensor` of shape `(batch_size * config.n_docs, config.max_combined_length)`, *optional*, returned when *output_retrieved=True*):
- Attention mask post-processed from the retrieved documents and the question encoder `input_ids` by
- the retriever.
+ Attention mask post-processed from the retrieved documents and the question encoder `input_ids` by the
+ retriever.
- If the model is not initialized with a `retriever` or `input_ids` is not given,
- `context_input_ids` and `context_attention_mask` have to be provided to the forward pass.
- They are returned by [`~RagRetriever.__call__`].
+ If the model is not initialized with a `retriever` or `input_ids` is not given, `context_input_ids` and
+ `context_attention_mask` have to be provided to the forward pass. They are returned by
+ [`~RagRetriever.__call__`].
doc_scores (`torch.FloatTensor` of shape `(batch_size, config.n_docs)`):
Score between each retrieved document embeddings (see `retrieved_doc_embeds`) and
`question_encoder_last_hidden_state`.
- If the model is not initialized with a `retriever` or `input_ids` is not given, `doc_scores`
- has to be provided to the forward pass. `doc_scores` are returned by
- [`~RagRetriever.__call__`].
+ If the model is not initialized with a `retriever` or `input_ids` is not given, `doc_scores` has to be
+ provided to the forward pass. `doc_scores` are returned by [`~RagRetriever.__call__`].
do_deduplication (`bool`, *optional*):
Whether or not to deduplicate the generations from different context documents for a given input. Has
to be set to `False` if used while training with distributed backend.
num_return_sequences(`int`, *optional*, defaults to 1):
The number of independently computed returned sequences for each element in the batch. Note that this
- is not the value we pass to the `generator`'s
- `[`~generation_utils.GenerationMixin.generate`]` function, where we set
- `num_return_sequences` to `num_beams`.
+ is not the value we pass to the `generator`'s `[`~generation_utils.GenerationMixin.generate`]`
+ function, where we set `num_return_sequences` to `num_beams`.
num_beams (`int`, *optional*, defaults to 1):
Number of beams for beam search. 1 means no beam search.
n_docs (`int`, *optional*, defaults to `config.n_docs`)
@@ -947,8 +953,8 @@ class RagSequenceForGeneration(RagPreTrainedModel):
Return:
`torch.LongTensor` of shape `(batch_size * num_return_sequences, sequence_length)`: The generated
- sequences. The second dimension (sequence length) is either equal to `max_length` or shorter if all
- batches finished early due to the `eos_token_id`.
+ sequences. The second dimension (sequence length) is either equal to `max_length` or shorter if all batches
+ finished early due to the `eos_token_id`.
"""
n_docs = n_docs if n_docs is not None else self.config.n_docs
@@ -1239,8 +1245,8 @@ class RagTokenForGeneration(RagPreTrainedModel):
If `True`, the logits are marginalized over all documents by making use of
`torch.nn.functional.log_softmax`.
reduce_loss (`bool`, *optional*):
- Only relevant if `labels` is passed. If `True`, the NLL loss is reduced using the
- `torch.Tensor.sum` operation.
+ Only relevant if `labels` is passed. If `True`, the NLL loss is reduced using the `torch.Tensor.sum`
+ operation.
kwargs (`Dict[str, any]`, optional, defaults to *{}*):
Legacy dictionary, which is required so that model can use *generate()* function.
@@ -1392,23 +1398,20 @@ class RagTokenForGeneration(RagPreTrainedModel):
Input IDs post-processed from the retrieved documents and the question encoder `input_ids` by the
retriever.
- If the model has is not initialized with a `retriever`, `context_input_ids` has to be provided
- to the forward pass. `context_input_ids` are returned by
- [`~RagRetriever.__call__`].
+ If the model has is not initialized with a `retriever`, `context_input_ids` has to be provided to the
+ forward pass. `context_input_ids` are returned by [`~RagRetriever.__call__`].
context_attention_mask (`torch.LongTensor` of shape `(batch_size * config.n_docs, config.max_combined_length)`, *optional*, returned when *output_retrieved=True*):
- Attention mask post-processed from the retrieved documents and the question encoder `input_ids` by
- the retriever.
+ Attention mask post-processed from the retrieved documents and the question encoder `input_ids` by the
+ retriever.
- If the model has is not initialized with a `retriever`, `context_input_ids` has to be provided
- to the forward pass. `context_input_ids` are returned by
- [`~RagRetriever.__call__`].
+ If the model has is not initialized with a `retriever`, `context_input_ids` has to be provided to the
+ forward pass. `context_input_ids` are returned by [`~RagRetriever.__call__`].
doc_scores (`torch.FloatTensor` of shape `(batch_size, config.n_docs)`):
Score between each retrieved document embeddings (see `retrieved_doc_embeds`) and
`question_encoder_last_hidden_state`.
- If the model has is not initialized with a `retriever`, `context_input_ids` has to be provided
- to the forward pass. `context_input_ids` are returned by
- [`~RagRetriever.__call__`].
+ If the model has is not initialized with a `retriever`, `context_input_ids` has to be provided to the
+ forward pass. `context_input_ids` are returned by [`~RagRetriever.__call__`].
max_length (`int`, *optional*, defaults to 20):
The maximum length of the sequence to be generated.
min_length (`int`, *optional*, defaults to 10):
@@ -1449,18 +1452,18 @@ class RagTokenForGeneration(RagPreTrainedModel):
enabled.
num_return_sequences(`int`, *optional*, defaults to 1):
The number of independently computed returned sequences for each element in the batch. Note that this
- is not the value we pass to the `generator`'s
- `[`~generation_utils.GenerationMixin.generate`] function, where we set `num_return_sequences` to `num_beams`. decoder_start_token_id (`int`, *optional*):
- If an encoder-decoder model starts decoding with a different token than *bos*, the id of that token.
+ is not the value we pass to the `generator`'s `[`~generation_utils.GenerationMixin.generate`] function,
+ where we set `num_return_sequences` to `num_beams`. decoder_start_token_id (`int`, *optional*): If an
+ encoder-decoder model starts decoding with a different token than *bos*, the id of that token.
n_docs (`int`, *optional*, defaults to `config.n_docs`)
Number of documents to retrieve and/or number of documents for which to generate an answer.
prefix_allowed_tokens_fn: (`Callable[[int, torch.Tensor], List[int]]`, *optional*):
If provided, this function constraints the beam search to allowed tokens only at each step. If not
provided no constraint is applied. This function takes 2 arguments `inputs_ids` and the batch ID
- `batch_id`. It has to return a list with the allowed tokens for the next generation step
- conditioned on the previously generated tokens `inputs_ids` and the batch ID `batch_id`. This
- argument is useful for constrained generation conditioned on the prefix, as described in
- [Autoregressive Entity Retrieval](https://arxiv.org/abs/2010.00904).
+ `batch_id`. It has to return a list with the allowed tokens for the next generation step conditioned on
+ the previously generated tokens `inputs_ids` and the batch ID `batch_id`. This argument is useful for
+ constrained generation conditioned on the prefix, as described in [Autoregressive Entity
+ Retrieval](https://arxiv.org/abs/2010.00904).
logits_processor (`LogitsProcessorList`, *optional*):
Custom logits processors that complement the default logits processors built from arguments and a
model's config. If a logit processor is passed that is already created with the arguments or a model's
@@ -1470,9 +1473,9 @@ class RagTokenForGeneration(RagPreTrainedModel):
model's config. If a stopping criteria is passed that is already created with the arguments or a
model's config an error is thrown.
forced_bos_token_id (`int`, *optional*):
- The id of the token to force as the first generated token after the `decoder_start_token_id`.
- Useful for multilingual models like [mBART](../model_doc/mbart) where the first generated token
- needs to be the target language token.
+ The id of the token to force as the first generated token after the `decoder_start_token_id`. Useful
+ for multilingual models like [mBART](../model_doc/mbart) where the first generated token needs to be
+ the target language token.
forced_eos_token_id (`int`, *optional*):
The id of the token to force as the last generated token when `max_length` is reached.
remove_invalid_values (`bool`, *optional*):
@@ -1481,8 +1484,8 @@ class RagTokenForGeneration(RagPreTrainedModel):
Return:
`torch.LongTensor` of shape `(batch_size * num_return_sequences, sequence_length)`: The generated
- sequences. The second dimension (sequence_length) is either equal to `max_length` or shorter if all
- batches finished early due to the `eos_token_id`.
+ sequences. The second dimension (sequence_length) is either equal to `max_length` or shorter if all batches
+ finished early due to the `eos_token_id`.
"""
# set default parameters
n_docs = n_docs if n_docs is not None else self.config.n_docs
diff --git a/src/transformers/models/rag/modeling_tf_rag.py b/src/transformers/models/rag/modeling_tf_rag.py
index 0b5ce2e103..948decc436 100644
--- a/src/transformers/models/rag/modeling_tf_rag.py
+++ b/src/transformers/models/rag/modeling_tf_rag.py
@@ -47,7 +47,8 @@ class TFRetrievAugLMMarginOutput(ModelOutput):
Prediction scores of the language modeling head. The score is possibly marginalized over all documents for
each vocabulary token.
past_key_values (`List[tf.Tensor]`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- List of `tf.Tensor` of length `config.n_layers`, with each tensor of shape `(2, batch_size, num_heads, sequence_length, embed_size_per_head)`).
+ List of `tf.Tensor` of length `config.n_layers`, with each tensor of shape `(2, batch_size, num_heads,
+ sequence_length, embed_size_per_head)`).
Contains precomputed hidden-states (key and values in the attention blocks) of the decoder that can be used
(see `past_key_values` input) to speed up sequential decoding.
@@ -55,8 +56,8 @@ class TFRetrievAugLMMarginOutput(ModelOutput):
Score between each retrieved document embeddings (see `retrieved_doc_embeds`) and
`question_encoder_last_hidden_state`.
retrieved_doc_embeds (`tf.Tensor` of shape `(batch_size, config.n_docs, hidden_size)`, *optional*, returned when *output_retrieved=True*):
- Embedded documents retrieved by the retriever. Is used with `question_encoder_last_hidden_state` to
- compute the `doc_scores`.
+ Embedded documents retrieved by the retriever. Is used with `question_encoder_last_hidden_state` to compute
+ the `doc_scores`.
retrieved_doc_ids (`tf.Tensor` (int32) of shape `(batch_size, config.n_docs)`, *optional*, returned when *output_retrieved=True*):
The indexes of the embedded documents retrieved by the retriever.
context_input_ids (`tf.Tensor`(int32) of shape `(batch_size * config.n_docs, config.max_combined_length)`, *optional*, returned when *output_retrieved=True*):
@@ -68,34 +69,37 @@ class TFRetrievAugLMMarginOutput(ModelOutput):
Sequence of hidden states at the output of the last layer of the question encoder pooled output of the
model.
question_enc_hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings and one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings and one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden states of the question encoder at the output of each layer plus the initial embedding outputs.
question_enc_attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the question encoder, after the attention softmax, used to compute the weighted
average in the self-attention heads.
generator_enc_last_hidden_state (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
Sequence of hidden-states at the output of the last layer of the generator encoder of the model.
generator_enc_hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings and one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings and one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden states of the generator encoder at the output of each layer plus the initial embedding outputs.
generator_enc_attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the generator encoder, after the attention softmax, used to compute the weighted
average in the self-attention heads.
generator_dec_hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings and one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings and one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden states of the generator decoder at the output of each layer plus the initial embedding outputs.
generator_dec_attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the generator decoder, after the attention softmax, used to compute the weighted
average in the self-attention heads.
@@ -127,7 +131,8 @@ class TFRetrievAugLMOutput(ModelOutput):
Prediction scores of the language modeling head. The score is possibly marginalized over all documents for
each vocabulary token.
past_key_values (`List[tf.Tensor]`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- List of `tf.Tensor` of length `config.n_layers`, with each tensor of shape `(2, batch_size, num_heads, sequence_length, embed_size_per_head)`).
+ List of `tf.Tensor` of length `config.n_layers`, with each tensor of shape `(2, batch_size, num_heads,
+ sequence_length, embed_size_per_head)`).
Contains precomputed hidden-states (key and values in the attention blocks) of the decoder that can be used
(see `past_key_values` input) to speed up sequential decoding.
@@ -135,8 +140,8 @@ class TFRetrievAugLMOutput(ModelOutput):
Score between each retrieved document embeddings (see `retrieved_doc_embeds`) and
`question_encoder_last_hidden_state`.
retrieved_doc_embeds (`tf.Tensor` of shape `(batch_size, config.n_docs, hidden_size)`, *optional*, returned when *output_retrieved=True*):
- Embedded documents retrieved by the retriever. Is used with `question_encoder_last_hidden_state` to
- compute the `doc_scores`.
+ Embedded documents retrieved by the retriever. Is used with `question_encoder_last_hidden_state` to compute
+ the `doc_scores`.
retrieved_doc_ids (`tf.Tensor` of shape `(batch_size, config.n_docs)`, *optional*, returned when *output_retrieved=True*):
The indexes of the embedded documents retrieved by the retriever.
context_input_ids (`tf.Tensor` of shape `(batch_size * config.n_docs, config.max_combined_length)`, *optional*, returned when *output_retrieved=True*):
@@ -148,34 +153,37 @@ class TFRetrievAugLMOutput(ModelOutput):
Sequence of hidden states at the output of the last layer of the question encoder pooled output of the
model.
question_enc_hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings and one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings and one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden states of the question encoder at the output of each layer plus the initial embedding outputs.
question_enc_attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the question encoder, after the attention softmax, used to compute the weighted
average in the self-attention heads.
generator_enc_last_hidden_state (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
Sequence of hidden-states at the output of the last layer of the generator encoder of the model.
generator_enc_hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings and one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings and one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden states of the generator encoder at the output of each layer plus the initial embedding outputs.
generator_enc_attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the generator encoder, after the attention softmax, used to compute the weighted
average in the self-attention heads.
generator_dec_hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings and one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings and one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden states of the generator decoder at the output of each layer plus the initial embedding outputs.
generator_dec_attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights of the generator decoder, after the attention softmax, used to compute the weighted
average in the self-attention heads.
@@ -363,18 +371,18 @@ RAG_START_DOCSTRING = r"""
relevant context documents. The documents are then prepended to the input. Such contextualized inputs is passed to
the generator.
- The question encoder can be any *autoencoding* model, preferably [`TFDPRQuestionEncoder`], and
- the generator can be any *seq2seq* model, preferably [`TFBartForConditionalGeneration`].
+ The question encoder can be any *autoencoding* model, preferably [`TFDPRQuestionEncoder`], and the generator can be
+ any *seq2seq* model, preferably [`TFBartForConditionalGeneration`].
- The model can be initialized with a [`RagRetriever`] for end-to-end generation or used in
- combination with the outputs of a retriever in multiple steps---see examples for more details. The model is
- compatible any *autoencoding* model as the `question_encoder` and any *seq2seq* model with language model head as
- the `generator`. It has been tested with [`TFDPRQuestionEncoder`] as the `question_encoder`
- and [`TFBartForConditionalGeneration`] as the `generator`.
+ The model can be initialized with a [`RagRetriever`] for end-to-end generation or used in combination with the
+ outputs of a retriever in multiple steps---see examples for more details. The model is compatible any
+ *autoencoding* model as the `question_encoder` and any *seq2seq* model with language model head as the `generator`.
+ It has been tested with [`TFDPRQuestionEncoder`] as the `question_encoder` and [`TFBartForConditionalGeneration`]
+ as the `generator`.
- This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading or saving, resizing the input
- embeddings, pruning heads etc.)
+ This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
This model is also a Tensorflow [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model)
subclass. Use it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to
@@ -400,9 +408,9 @@ RAG_START_DOCSTRING = r"""
RAG_FORWARD_INPUTS_DOCSTRING = r"""
Args:
input_ids (`tf.Tensor` of shape `(batch_size, sequence_length)`):
- Indices of input sequence tokens in the vocabulary. [`RagConfig`], used to initialize
- the model, specifies which generator to use, it also specifies a compatible generator tokenizer. Use that
- tokenizer class to obtain the indices.
+ Indices of input sequence tokens in the vocabulary. [`RagConfig`], used to initialize the model, specifies
+ which generator to use, it also specifies a compatible generator tokenizer. Use that tokenizer class to
+ obtain the indices.
attention_mask (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
@@ -412,41 +420,41 @@ RAG_FORWARD_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
encoder_outputs (`tuple(tuple(tf.Tensor)`, *optional*)
Tuple consists of (`generator_enc_last_hidden_state`, *optional*: `generator_enc_hidden_states`,
- *optional*: `generator_enc_attentions`). `generator_enc_last_hidden_state` of shape
- `(batch_size, n_docs * sequence_length, hidden_size)` is a sequence of hidden-states at the output of
- the last layer of the generator's encoder.
+ *optional*: `generator_enc_attentions`). `generator_enc_last_hidden_state` of shape `(batch_size, n_docs *
+ sequence_length, hidden_size)` is a sequence of hidden-states at the output of the last layer of the
+ generator's encoder.
Used by the ([`TFRagModel`]) model during decoding.
decoder_input_ids (`tf.Tensor` of shape `(batch_size, target_sequence_length)`, *optional*):
Provide for generation tasks. *None* by default, construct as per instructions for the generator model
you're using with your RAG instance.
decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
- Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will
- also be used by default.
+ Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
+ be used by default.
past_key_values (`tuple(tuple(tf.Tensor))`):
Tuple consists of two elements: `encoder_outputs` of the RAG model (see `encoder_outputs`) and
- `past_key_values` of the underlying generator. Can be used to speed up decoding.
- `past_key_values` are used in the ([`RagTokenForGeneration`]) model during
- decoding.
+ `past_key_values` of the underlying generator. Can be used to speed up decoding. `past_key_values` are used
+ in the ([`RagTokenForGeneration`]) model during decoding.
doc_scores (`tf.Tensor` of shape `(batch_size, config.n_docs)`):
Score between each retrieved document embeddings (see `retrieved_doc_embeds`) and
- `question_encoder_last_hidden_state`. If the model has is not initialized with a `retriever`
- `doc_scores` has to be provided to the forward pass. `doc_scores` can be computed via
- `question_encoder_last_hidden_state` and `retrieved_doc_embeds`, see examples for more
- information.
+ `question_encoder_last_hidden_state`. If the model has is not initialized with a `retriever` `doc_scores`
+ has to be provided to the forward pass. `doc_scores` can be computed via
+ `question_encoder_last_hidden_state` and `retrieved_doc_embeds`, see examples for more information.
context_input_ids (`tf.Tensor` of shape `(batch_size * config.n_docs, config.max_combined_length)`, *optional*, returned when *output_retrieved=True*):
Input IDs post-processed from the retrieved documents and the question encoder `input_ids` by the
retriever.
- If the model has is not initialized with a `retriever` ``context_input_ids` has to be provided to the forward pass. `context_input_ids` are returned by [`~RagRetriever.__call__`]. context_attention_mask (`tf.Tensor` of shape `(batch_size * config.n_docs, config.max_combined_length)`, *optional*, returned when *output_retrieved=True*): Attention mask post-processed from the retrieved documents and the question encoder `input_ids` by the
- retriever.
+ If the model has is not initialized with a `retriever` ``context_input_ids` has to be provided to the
+ forward pass. `context_input_ids` are returned by [`~RagRetriever.__call__`]. context_attention_mask
+ (`tf.Tensor` of shape `(batch_size * config.n_docs, config.max_combined_length)`, *optional*, returned when
+ *output_retrieved=True*): Attention mask post-processed from the retrieved documents and the question
+ encoder `input_ids` by the retriever.
- If the model has is not initialized with a `retriever` `context_attention_mask` has to be provided
- to the forward pass. `context_attention_mask` are returned by
- [`~RagRetriever.__call__`].
+ If the model has is not initialized with a `retriever` `context_attention_mask` has to be provided to the
+ forward pass. `context_attention_mask` are returned by [`~RagRetriever.__call__`].
use_cache (`bool`, *optional*, defaults to `True`):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`).
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`).
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -454,8 +462,8 @@ RAG_FORWARD_INPUTS_DOCSTRING = r"""
Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
more detail.
output_retrieved(`bool`, *optional*):
- Whether or not to return the `retrieved_doc_embeds`, `retrieved_doc_ids`,
- `context_input_ids` and `context_attention_mask`. See returned tensors for more detail.
+ Whether or not to return the `retrieved_doc_embeds`, `retrieved_doc_ids`, `context_input_ids` and
+ `context_attention_mask`. See returned tensors for more detail.
return_dict (`bool`, *optional*):
Whether or not to return a [`TFRetrievAugLMOutput`] instead of a plain tuple.
n_docs (`int`, *optional*, defaults to `config.n_docs``)
@@ -829,7 +837,8 @@ class TFRagTokenForGeneration(TFRagPreTrainedModel, TFCausalLanguageModelingLoss
Output:
mimic of torch_tensor.index_select(dim, indices)
- credit: https://stackoverflow.com/questions/58464790/is-there-an-equivalent-function-of-pytorch-named-index-select-in-tensorflow
+ credit:
+ https://stackoverflow.com/questions/58464790/is-there-an-equivalent-function-of-pytorch-named-index-select-in-tensorflow
"""
shape = shape_list(input_)
if dim == -1:
@@ -1078,23 +1087,20 @@ class TFRagTokenForGeneration(TFRagPreTrainedModel, TFCausalLanguageModelingLoss
Input IDs post-processed from the retrieved documents and the question encoder `input_ids` by the
retriever.
- If the model has is not initialized with a `retriever`, `context_input_ids` has to be provided
- to the forward pass. `context_input_ids` are returned by
- [`~RagRetriever.__call__`].
+ If the model has is not initialized with a `retriever`, `context_input_ids` has to be provided to the
+ forward pass. `context_input_ids` are returned by [`~RagRetriever.__call__`].
context_attention_mask (`tf.Tensor` of shape `(batch_size * config.n_docs, config.max_combined_length)`, *optional*, returned when *output_retrieved=True*):
- Attention mask post-processed from the retrieved documents and the question encoder `input_ids` by
- the retriever.
+ Attention mask post-processed from the retrieved documents and the question encoder `input_ids` by the
+ retriever.
- If the model has is not initialized with a `retriever`, `context_input_ids` has to be provided
- to the forward pass. `context_input_ids` are returned by
- [`~RagRetriever.__call__`].
+ If the model has is not initialized with a `retriever`, `context_input_ids` has to be provided to the
+ forward pass. `context_input_ids` are returned by [`~RagRetriever.__call__`].
doc_scores (`tf.Tensor` of shape `(batch_size, config.n_docs)`):
Score between each retrieved document embeddings (see `retrieved_doc_embeds`) and
`question_encoder_last_hidden_state`.
- If the model has is not initialized with a `retriever`, `context_input_ids` has to be provided
- to the forward pass. `context_input_ids` are returned by
- [`~RagRetriever.__call__`].
+ If the model has is not initialized with a `retriever`, `context_input_ids` has to be provided to the
+ forward pass. `context_input_ids` are returned by [`~RagRetriever.__call__`].
max_length (`int`, *optional*, defaults to 20):
The maximum length of the sequence to be generated.
min_length (`int`, *optional*, defaults to 10):
@@ -1125,9 +1131,9 @@ class TFRagTokenForGeneration(TFRagPreTrainedModel, TFCausalLanguageModelingLoss
Number of beams for beam search. 1 means no beam search.
num_return_sequences(`int`, *optional*, defaults to 1):
The number of independently computed returned sequences for each element in the batch. Note that this
- is not the value we pass to the `generator`'s
- `[`~generation_utils.GenerationMixin.generate`] function, where we set `num_return_sequences` to `num_beams`. decoder_start_token_id (`int`, *optional*):
- If an encoder-decoder model starts decoding with a different token than *bos*, the id of that token.
+ is not the value we pass to the `generator`'s `[`~generation_utils.GenerationMixin.generate`] function,
+ where we set `num_return_sequences` to `num_beams`. decoder_start_token_id (`int`, *optional*): If an
+ encoder-decoder model starts decoding with a different token than *bos*, the id of that token.
n_docs (`int`, *optional*, defaults to `config.n_docs`)
Number of documents to retrieve and/or number of documents for which to generate an answer.
output_attentions (`bool`, *optional*, defaults to *False*):
@@ -1144,9 +1150,9 @@ class TFRagTokenForGeneration(TFRagPreTrainedModel, TFCausalLanguageModelingLoss
Additional model specific kwargs will be forwarded to the `forward` function of the model.
Return:
- `tf.Tensor` of shape `(batch_size * num_return_sequences, sequence_length)`: The generated
- sequences. The second dimension (sequence_length) is either equal to `max_length` or shorter if all
- batches finished early due to the `eos_token_id`.
+ `tf.Tensor` of shape `(batch_size * num_return_sequences, sequence_length)`: The generated sequences. The
+ second dimension (sequence_length) is either equal to `max_length` or shorter if all batches finished early
+ due to the `eos_token_id`.
"""
# set default parameters
n_docs = n_docs if n_docs is not None else self.config.n_docs
@@ -1493,8 +1499,8 @@ class TFRagSequenceForGeneration(TFRagPreTrainedModel, TFCausalLanguageModelingL
):
r"""
exclude_bos_score (`bool`, *optional*):
- Only relevant if `labels` is passed. If `True`, the score of the BOS token is disregarded when
- computing the loss.
+ Only relevant if `labels` is passed. If `True`, the score of the BOS token is disregarded when computing
+ the loss.
labels (`tf.Tensor` or `np.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
Labels for computing the cross entropy classification loss according to Rag-Sequence model formulation See
https://arxiv.org/pdf/2005.11401.pdf Section 2.1 for details about Rag-Sequence formulation. Indices should
@@ -1717,38 +1723,37 @@ class TFRagSequenceForGeneration(TFRagPreTrainedModel, TFCausalLanguageModelingL
**model_kwargs
):
"""
- Implements RAG sequence "thorough" decoding. Read the
- [`~generation_utils.GenerationMixin.generate`]` documentation for more information on how to
- set other generate input parameters
+ Implements RAG sequence "thorough" decoding. Read the [`~generation_utils.GenerationMixin.generate`]`
+ documentation for more information on how to set other generate input parameters
Args:
input_ids (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
The sequence used as a prompt for the generation. If `input_ids` is not passed, then
`context_input_ids` has to be provided.
attention_mask (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
- Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1
- for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask)
+ Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for
+ tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention
+ masks?](../glossary#attention-mask)
context_input_ids (`tf.Tensor` of shape `(batch_size * config.n_docs, config.max_combined_length)`, *optional*, returned when *output_retrieved=True*):
Input IDs post-processed from the retrieved documents and the question encoder input_ids by the
retriever.
context_attention_mask (`tf.Tensor` of shape `(batch_size * config.n_docs, config.max_combined_length)`, *optional*, returned when *output_retrieved=True*):
- Attention mask post-processed from the retrieved documents and the question encoder `input_ids` by
- the retriever. If the model has is not initialized with a `retriever` or `input_ids` is not given,
- `context_input_ids` and `context_attention_mask` have to be provided to the forward pass.
- They are returned by [`~RagRetriever.__call__`].
+ Attention mask post-processed from the retrieved documents and the question encoder `input_ids` by the
+ retriever. If the model has is not initialized with a `retriever` or `input_ids` is not given,
+ `context_input_ids` and `context_attention_mask` have to be provided to the forward pass. They are
+ returned by [`~RagRetriever.__call__`].
doc_scores (`tf.Tensor` of shape `(batch_size, config.n_docs)`):
Score between each retrieved document embeddings (see `retrieved_doc_embeds`) and
`question_encoder_last_hidden_state`. If the model has is not initialized with a `retriever` or
- `input_ids` is not given, `doc_scores` has to be provided to the forward pass. `doc_scores`
- are returned by [`~RagRetriever.__call__`].
+ `input_ids` is not given, `doc_scores` has to be provided to the forward pass. `doc_scores` are
+ returned by [`~RagRetriever.__call__`].
do_deduplication (`bool`, *optional*):
Whether or not to deduplicate the generations from different context documents for a given input. Has
to be set to `False` if used while training with distributed backend.
num_return_sequences(`int`, *optional*, defaults to 1):
The number of independently computed returned sequences for each element in the batch. Note that this
- is not the value we pass to the `generator`'s
- `[`~generation_utils.GenerationMixin.generate`]` function, where we set
- `num_return_sequences` to `num_beams`.
+ is not the value we pass to the `generator`'s `[`~generation_utils.GenerationMixin.generate`]`
+ function, where we set `num_return_sequences` to `num_beams`.
num_beams (`int`, *optional*, defaults to 1):
Number of beams for beam search. 1 means no beam search.
n_docs (`int`, *optional*, defaults to `config.n_docs`)
@@ -1757,9 +1762,9 @@ class TFRagSequenceForGeneration(TFRagPreTrainedModel, TFCausalLanguageModelingL
Additional kwargs will be passed to [`~generation_utils.GenerationMixin.generate`]
Return:
- `tf.Tensor` of shape `(batch_size * num_return_sequences, sequence_length)`: The generated
- sequences. The second dimension (sequence length) is either equal to `max_length` or shorter if all
- batches finished early due to the `eos_token_id`.
+ `tf.Tensor` of shape `(batch_size * num_return_sequences, sequence_length)`: The generated sequences. The
+ second dimension (sequence length) is either equal to `max_length` or shorter if all batches finished early
+ due to the `eos_token_id`.
"""
n_docs = n_docs if n_docs is not None else self.config.n_docs
diff --git a/src/transformers/models/rag/retrieval_rag.py b/src/transformers/models/rag/retrieval_rag.py
index 2c2820b084..db6aef4940 100644
--- a/src/transformers/models/rag/retrieval_rag.py
+++ b/src/transformers/models/rag/retrieval_rag.py
@@ -68,9 +68,8 @@ class Index:
The number of docs retrieved per query.
Returns:
- `np.ndarray` of shape `(batch_size, n_docs)`: A tensor of indices of retrieved documents.
- `np.ndarray` of shape `(batch_size, vector_size)`: A tensor of vector representations of
- retrieved documents.
+ `np.ndarray` of shape `(batch_size, n_docs)`: A tensor of indices of retrieved documents. `np.ndarray` of
+ shape `(batch_size, vector_size)`: A tensor of vector representations of retrieved documents.
"""
raise NotImplementedError
@@ -98,8 +97,7 @@ class LegacyIndex(Index):
vector_size (`int`):
The dimension of indexed vectors.
index_path (`str`):
- A path to a *directory* containing index files compatible with
- [`~models.rag.retrieval_rag.LegacyIndex`]
+ A path to a *directory* containing index files compatible with [`~models.rag.retrieval_rag.LegacyIndex`]
"""
INDEX_FILENAME = "hf_bert_base.hnswSQ8_correct_phi_128.c_index"
@@ -228,9 +226,9 @@ class HFIndexBase(Index):
class CanonicalHFIndex(HFIndexBase):
"""
- A wrapper around an instance of [`~datasets.Datasets`]. If `index_path` is set to `None`, we load the
- pre-computed index available with the [`~datasets.arrow_dataset.Dataset`], otherwise, we load the index from
- the indicated path on disk.
+ A wrapper around an instance of [`~datasets.Datasets`]. If `index_path` is set to `None`, we load the pre-computed
+ index available with the [`~datasets.arrow_dataset.Dataset`], otherwise, we load the index from the indicated path
+ on disk.
Args:
vector_size (`int`): the dimension of the passages embeddings used by the index
@@ -240,11 +238,12 @@ class CanonicalHFIndex(HFIndexBase):
dataset_split (`str`, optional, defaults to `train`)
Which split of the `dataset` to load.
index_name (`str`, optional, defaults to `train`)
- The index_name of the index associated with the `dataset`. The index loaded from `index_path` will be
- saved under this name.
+ The index_name of the index associated with the `dataset`. The index loaded from `index_path` will be saved
+ under this name.
index_path (`str`, optional, defaults to `None`)
The path to the serialized faiss index on disk.
- use_dummy_dataset (`bool`, optional, defaults to `False`): If True, use the dummy configuration of the dataset for tests.
+ use_dummy_dataset (`bool`, optional, defaults to `False`):
+ If True, use the dummy configuration of the dataset for tests.
"""
def __init__(
@@ -331,8 +330,8 @@ class RagRetriever:
Args:
config ([`RagConfig`]):
The configuration of the RAG model this Retriever is used with. Contains parameters indicating which
- `Index` to build. You can load your own custom dataset with `config.index_name="custom"` or use a
- canonical one (default) from the datasets library with `config.index_name="wiki_dpr"` for example.
+ `Index` to build. You can load your own custom dataset with `config.index_name="custom"` or use a canonical
+ one (default) from the datasets library with `config.index_name="wiki_dpr"` for example.
question_encoder_tokenizer ([`PreTrainedTokenizer`]):
The tokenizer that was used to tokenize the question. It is used to decode the question and then use the
generator_tokenizer.
@@ -537,10 +536,9 @@ class RagRetriever:
Return:
`Tuple[np.ndarray, np.ndarray, List[dict]]`: A tuple with the following objects:
- - **retrieved_doc_embeds** (`np.ndarray` of shape `(batch_size, n_docs, dim)`) -- The retrieval
- embeddings of the retrieved docs per query.
- - **doc_ids** (`np.ndarray` of shape `(batch_size, n_docs)`) -- The ids of the documents in the
- index
+ - **retrieved_doc_embeds** (`np.ndarray` of shape `(batch_size, n_docs, dim)`) -- The retrieval embeddings
+ of the retrieved docs per query.
+ - **doc_ids** (`np.ndarray` of shape `(batch_size, n_docs)`) -- The ids of the documents in the index
- **doc_dicts** (`List[dict]`): The `retrieved_doc_embeds` examples per query.
"""
@@ -578,8 +576,7 @@ class RagRetriever:
- `'pt'`: Return PyTorch `torch.Tensor` objects.
- `'np'`: Return Numpy `np.ndarray` objects.
- Returns: [`BatchEncoding`]: A [`BatchEncoding`] with the following
- fields:
+ Returns: [`BatchEncoding`]: A [`BatchEncoding`] with the following fields:
- **context_input_ids** -- List of token ids to be fed to a model.
diff --git a/src/transformers/models/reformer/configuration_reformer.py b/src/transformers/models/reformer/configuration_reformer.py
index aaad9e96d6..58df9525c9 100755
--- a/src/transformers/models/reformer/configuration_reformer.py
+++ b/src/transformers/models/reformer/configuration_reformer.py
@@ -13,7 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" Reformer model configuration """
+""" Reformer model configuration"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging
@@ -29,11 +29,11 @@ REFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class ReformerConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`ReformerModel`]. It is used to
- instantiate a Reformer model according to the specified arguments, defining the model architecture.
+ This is the configuration class to store the configuration of a [`ReformerModel`]. It is used to instantiate a
+ Reformer model according to the specified arguments, defining the model architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
attention_head_size (`int`, *optional*, defaults to 64):
@@ -42,8 +42,8 @@ class ReformerConfig(PretrainedConfig):
List of attention layer types in ascending order. It can be chosen between a LSHSelfAttention layer
(`"lsh"`) and a LocalSelfAttention layer (`"local"`).
- For more information on LSHSelfAttention layer, see [LSH Self Attention](reformer#lsh-self-attention). For more information on LocalSelfAttention layer, see [Local Self
- Attention](reformer#local-self-attention).
+ For more information on LSHSelfAttention layer, see [LSH Self Attention](reformer#lsh-self-attention). For
+ more information on LocalSelfAttention layer, see [Local Self Attention](reformer#local-self-attention).
axial_pos_embds (`bool`, *optional*, defaults to `True`):
Whether or not to use axial position embeddings. For more information on how axial position embeddings
work, see [Axial Position Encodings](reformer#axial-positional-encodings).
@@ -54,26 +54,29 @@ class ReformerConfig(PretrainedConfig):
The position dims of the axial position encodings. During training, the product of the position dims has to
be equal to the sequence length.
- For more information on how axial position embeddings work, see [Axial Position Encodings](reformer#axial-positional-encodings).
+ For more information on how axial position embeddings work, see [Axial Position
+ Encodings](reformer#axial-positional-encodings).
axial_pos_embds_dim (`List[int]`, *optional*, defaults to `[64, 192]`):
The embedding dims of the axial position encodings. The sum of the embedding dims has to be equal to the
hidden size.
- For more information on how axial position embeddings work, see [Axial Position Encodings](reformer#axial-positional-encodings).
+ For more information on how axial position embeddings work, see [Axial Position
+ Encodings](reformer#axial-positional-encodings).
chunk_size_lm_head (`int`, *optional*, defaults to 0):
The chunk size of the final language model feed forward head layer. A chunk size of 0 means that the feed
forward layer is not chunked. A chunk size of n means that the feed forward layer processes n <
sequence_length embeddings at a time.
- For more information on feed forward chunking, see [How does Feed Forward Chunking work?](../glossary#feed-forward-chunking).
+ For more information on feed forward chunking, see [How does Feed Forward Chunking
+ work?](../glossary#feed-forward-chunking).
eos_token_id (`int`, *optional*, defaults to 2):
The token id for the end-of-sentence token.
feed_forward_size (`int`, *optional*, defaults to 512):
Dimensionality of the feed_forward layer in the residual attention block.
hash_seed (`int`, *optional*):
- Seed that can be used to make local sensitive hashing in `LSHSelfAttention` deterministic. This should
- only be set for testing purposed. For evaluation and training purposes `hash_seed` should be left as
- `None` to ensure fully random rotations in local sensitive hashing scheme.
+ Seed that can be used to make local sensitive hashing in `LSHSelfAttention` deterministic. This should only
+ be set for testing purposed. For evaluation and training purposes `hash_seed` should be left as `None` to
+ ensure fully random rotations in local sensitive hashing scheme.
hidden_act (`str` or `Callable`, *optional*, defaults to `"relu"`):
The non-linear activation function (function or string) in the feed forward layer in the residual attention
block. If string, `"gelu"`, `"relu"`, `"silu"` and `"gelu_new"` are supported.
@@ -84,20 +87,18 @@ class ReformerConfig(PretrainedConfig):
initializer_range (`float`, *optional*, defaults to 0.02):
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
is_decoder (`bool`, *optional*, defaults to `False`):
- Whether or not to use a causal mask in addition to the `attention_mask` passed to
- [`ReformerModel`]. When using the Reformer for causal language modeling, this argument
- should be set to `True`.
+ Whether or not to use a causal mask in addition to the `attention_mask` passed to [`ReformerModel`]. When
+ using the Reformer for causal language modeling, this argument should be set to `True`.
layer_norm_eps (`float`, *optional*, defaults to 1e-12):
The epsilon used by the layer normalization layers.
local_chunk_length (`int`, *optional*, defaults to 64):
- Length of chunk which attends to itself in `LocalSelfAttention`. Chunking reduces memory complexity
- from sequence length x sequence length (self attention) to chunk length x chunk length x sequence length /
- chunk length (chunked self attention).
+ Length of chunk which attends to itself in `LocalSelfAttention`. Chunking reduces memory complexity from
+ sequence length x sequence length (self attention) to chunk length x chunk length x sequence length / chunk
+ length (chunked self attention).
local_num_chunks_before (`int`, *optional*, defaults to 1):
Number of previous neighbouring chunks to attend to in `LocalSelfAttention` layer to itself.
local_num_chunks_after (`int`, *optional*, defaults to 0):
- Number of following neighbouring chunks to attend to in `LocalSelfAttention` layer in addition to
- itself.
+ Number of following neighbouring chunks to attend to in `LocalSelfAttention` layer in addition to itself.
local_attention_probs_dropout_prob (`float`, *optional*, defaults to 0.1):
The dropout ratio for the attention probabilities in `LocalSelfAttention`.
lsh_attn_chunk_length (`int`, *optional*, defaults to 64):
@@ -117,16 +118,15 @@ class ReformerConfig(PretrainedConfig):
Number of attention heads for each attention layer in the Transformer encoder.
num_buckets (`int` or `List[int]`, *optional*):
Number of buckets, the key query vectors can be "hashed into" using the locality sensitive hashing scheme.
- Each query key vector is hashed into a hash in `1, ..., num_buckets`. The number of buckets can also
- be factorized into a list for improved memory complexity. In this case, each query key vector is hashed
- into a hash in `1-1, 1-2, ..., num_buckets[0]-1, ..., num_buckets[0]-num_buckets[1]` if
- `num_buckets` is factorized into two factors. The number of buckets (or the product the factors)
- should approximately equal sequence length / lsh_chunk_length. If `num_buckets` not set, a good value
- is calculated on the fly.
+ Each query key vector is hashed into a hash in `1, ..., num_buckets`. The number of buckets can also be
+ factorized into a list for improved memory complexity. In this case, each query key vector is hashed into a
+ hash in `1-1, 1-2, ..., num_buckets[0]-1, ..., num_buckets[0]-num_buckets[1]` if `num_buckets` is
+ factorized into two factors. The number of buckets (or the product the factors) should approximately equal
+ sequence length / lsh_chunk_length. If `num_buckets` not set, a good value is calculated on the fly.
num_hashes (`int`, *optional*, defaults to 1):
Number of hashing rounds (e.g., number of random rotations) in Local Sensitive Hashing scheme. The higher
- `num_hashes`, the more accurate the `LSHSelfAttention` becomes, but also the more memory and time
- intensive the hashing becomes.
+ `num_hashes`, the more accurate the `LSHSelfAttention` becomes, but also the more memory and time intensive
+ the hashing becomes.
pad_token_id (`int`, *optional*, defaults to 0):
The token id for the padding token.
vocab_size (`int`, *optional*, defaults to 320):\
diff --git a/src/transformers/models/reformer/modeling_reformer.py b/src/transformers/models/reformer/modeling_reformer.py
index 71fef5a7c7..3f42d74ca2 100755
--- a/src/transformers/models/reformer/modeling_reformer.py
+++ b/src/transformers/models/reformer/modeling_reformer.py
@@ -13,7 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-"""PyTorch REFORMER model. """
+"""PyTorch REFORMER model."""
import sys
from collections import namedtuple
@@ -1809,22 +1809,23 @@ class ReformerModelOutput(ModelOutput):
last_hidden_state (`torch.FloatTensor` of shape `(batch_size, num_predict, hidden_size)`):
Sequence of hidden-states at the last layer of the model.
- `num_predict` corresponds to `target_mapping.shape[1]`. If `target_mapping` is `None`, then
- `num_predict` corresponds to `sequence_length`.
+ `num_predict` corresponds to `target_mapping.shape[1]`. If `target_mapping` is `None`, then `num_predict`
+ corresponds to `sequence_length`.
past_buckets_states (`List[Tuple(torch.LongTensor, torch.FloatTensor)]`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- List of `Tuple(torch.LongTensor, torch.FloatTensor` of length `config.n_layers`, with the first
- element being the previous *buckets* of shape `(batch_size, num_heads, num_hashes, sequence_length)`)
- and the second being the previous *hidden_states* of shape `(batch_size, sequence_length, hidden_size)`).
+ List of `Tuple(torch.LongTensor, torch.FloatTensor` of length `config.n_layers`, with the first element
+ being the previous *buckets* of shape `(batch_size, num_heads, num_hashes, sequence_length)`) and the
+ second being the previous *hidden_states* of shape `(batch_size, sequence_length, hidden_size)`).
- Contains precomputed buckets and hidden-states that can be used (see `past_buckets_states` input) to
- speed up sequential decoding.
+ Contains precomputed buckets and hidden-states that can be used (see `past_buckets_states` input) to speed
+ up sequential decoding.
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings and one for the output of each
- layer) of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings and one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -1847,22 +1848,23 @@ class ReformerModelWithLMHeadOutput(ModelOutput):
logits (`torch.FloatTensor` of shape `(batch_size, num_predict, config.vocab_size)`):
Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
- `num_predict` corresponds to `target_mapping.shape[1]`. If `target_mapping` is `None`, then
- `num_predict` corresponds to `sequence_length`.
+ `num_predict` corresponds to `target_mapping.shape[1]`. If `target_mapping` is `None`, then `num_predict`
+ corresponds to `sequence_length`.
past_buckets_states (`List[Tuple(torch.LongTensor, torch.FloatTensor)]`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- List of `Tuple(torch.LongTensor, torch.FloatTensor` of length `config.n_layers`, with the first
- element being the previous *buckets* of shape `(batch_size, num_heads, num_hashes, sequence_length)`)
- and the second being the previous *hidden_states* of shape `(batch_size, sequence_length, hidden_size)`).
+ List of `Tuple(torch.LongTensor, torch.FloatTensor` of length `config.n_layers`, with the first element
+ being the previous *buckets* of shape `(batch_size, num_heads, num_hashes, sequence_length)`) and the
+ second being the previous *hidden_states* of shape `(batch_size, sequence_length, hidden_size)`).
- Contains precomputed buckets and hidden-states that can be used (see `past_buckets_states` input) to
- speed up sequential decoding.
+ Contains precomputed buckets and hidden-states that can be used (see `past_buckets_states` input) to speed
+ up sequential decoding.
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- TTuple of `torch.FloatTensor` (one for the output of the embeddings and one for the output of each
- layer) of shape `(batch_size, sequence_length, hidden_size)`.
+ TTuple of `torch.FloatTensor` (one for the output of the embeddings and one for the output of each layer)
+ of shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -1876,22 +1878,21 @@ class ReformerModelWithLMHeadOutput(ModelOutput):
REFORMER_START_DOCSTRING = r"""
- Reformer was proposed in [Reformer: The Efficient Transformer](https://arxiv.org/abs/2001.04451) by Nikita
- Kitaev, Łukasz Kaiser, Anselm Levskaya.
+ Reformer was proposed in [Reformer: The Efficient Transformer](https://arxiv.org/abs/2001.04451) by Nikita Kitaev,
+ Łukasz Kaiser, Anselm Levskaya.
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
Parameters:
config ([`ReformerConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
REFORMER_INPUTS_DOCSTRING = r"""
@@ -1901,9 +1902,8 @@ REFORMER_INPUTS_DOCSTRING = r"""
a multiple of the relevant model's chunk lengths (lsh's, local's or both). During evaluation, the indices
are automatically padded to be a multiple of the chunk length.
- Indices can be obtained using [`ReformerTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`ReformerTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -1914,7 +1914,8 @@ REFORMER_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
@@ -1924,24 +1925,24 @@ REFORMER_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
num_hashes (`int`, *optional*):
The number of hashing rounds that should be performed during bucketing. Setting this argument overwrites
the default defined in `config.num_hashes`.
For more information, see `num_hashes` in [`ReformerConfig`].
past_buckets_states (`List[Tuple(torch.LongTensor, torch.FloatTensor)]`, *optional*):
- List of `Tuple(torch.LongTensor, torch.FloatTensor` of length `config.n_layers`, with the first
- element being the previous *buckets* of shape `(batch_size, num_heads, num_hashes, sequence_length)`)
- and the second being the previous *hidden_states* of shape `(batch_size, sequence_length, hidden_size)`).
+ List of `Tuple(torch.LongTensor, torch.FloatTensor` of length `config.n_layers`, with the first element
+ being the previous *buckets* of shape `(batch_size, num_heads, num_hashes, sequence_length)`) and the
+ second being the previous *hidden_states* of shape `(batch_size, sequence_length, hidden_size)`).
Contains precomputed hidden-states and buckets (only relevant for LSH Self-Attention). Can be used to speed
up sequential decoding.
use_cache (`bool`, *optional*):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`).
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`).
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -2168,7 +2169,7 @@ class ReformerModel(ReformerPreTrainedModel):
return input_ids, inputs_embeds, attention_mask, position_ids, input_shape
-@add_start_docstrings("""Reformer Model with a `language modeling` head on top. """, REFORMER_START_DOCSTRING)
+@add_start_docstrings("""Reformer Model with a `language modeling` head on top.""", REFORMER_START_DOCSTRING)
class ReformerModelWithLMHead(ReformerPreTrainedModel):
def __init__(self, config):
super().__init__(config)
@@ -2216,8 +2217,9 @@ class ReformerModelWithLMHead(ReformerPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[-100, 0, ..., config.vocab_size - 1]`. All labels set to `-100` are ignored (masked), the loss is only
- computed for labels in `[0, ..., config.vocab_size]`
+ Labels for computing the sequence classification/regression loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size - 1]`. All labels set to `-100` are ignored (masked), the loss is only computed for
+ labels in `[0, ..., config.vocab_size]`
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -2288,7 +2290,7 @@ class ReformerModelWithLMHead(ReformerPreTrainedModel):
return reord_past_buckets_states
-@add_start_docstrings("""Reformer Model with a `language modeling` head on top. """, REFORMER_START_DOCSTRING)
+@add_start_docstrings("""Reformer Model with a `language modeling` head on top.""", REFORMER_START_DOCSTRING)
class ReformerForMaskedLM(ReformerPreTrainedModel):
def __init__(self, config):
super().__init__(config)
@@ -2329,8 +2331,9 @@ class ReformerForMaskedLM(ReformerPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked),
+ the loss is only computed for the tokens with labels
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -2410,8 +2413,9 @@ class ReformerForSequenceClassification(ReformerPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -2530,12 +2534,12 @@ class ReformerForQuestionAnswering(ReformerPreTrainedModel):
r"""
start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the start of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the end of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
diff --git a/src/transformers/models/reformer/tokenization_reformer.py b/src/transformers/models/reformer/tokenization_reformer.py
index cfd6fa6651..83377622f8 100644
--- a/src/transformers/models/reformer/tokenization_reformer.py
+++ b/src/transformers/models/reformer/tokenization_reformer.py
@@ -47,8 +47,8 @@ class ReformerTokenizer(PreTrainedTokenizer):
"""
Construct a Reformer tokenizer. Based on [SentencePiece](https://github.com/google/sentencepiece) .
- This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods.
- Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
+ this superclass for more information regarding those methods.
Args:
vocab_file (`str`):
@@ -59,8 +59,8 @@ class ReformerTokenizer(PreTrainedTokenizer):
- When building a sequence using special tokens, this is not the token that is used for the end of
- sequence. The token used is the `sep_token`.
+ When building a sequence using special tokens, this is not the token that is used for the end of sequence.
+ The token used is the `sep_token`.
@@ -72,7 +72,9 @@ class ReformerTokenizer(PreTrainedTokenizer):
additional_special_tokens (`List[str]`, *optional*):
Additional special tokens used by the tokenizer.
sp_model_kwargs (`dict`, *optional*):
- Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things, to set:
+ Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for
+ SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things,
+ to set:
- `enable_sampling`: Enable subword regularization.
- `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout.
diff --git a/src/transformers/models/reformer/tokenization_reformer_fast.py b/src/transformers/models/reformer/tokenization_reformer_fast.py
index f466e69837..86218eb73c 100644
--- a/src/transformers/models/reformer/tokenization_reformer_fast.py
+++ b/src/transformers/models/reformer/tokenization_reformer_fast.py
@@ -53,10 +53,11 @@ PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
class ReformerTokenizerFast(PreTrainedTokenizerFast):
"""
- Construct a "fast" Reformer tokenizer (backed by HuggingFace's *tokenizers* library). Based on [Unigram](https://huggingface.co/docs/tokenizers/python/latest/components.html?highlight=unigram#models).
+ Construct a "fast" Reformer tokenizer (backed by HuggingFace's *tokenizers* library). Based on
+ [Unigram](https://huggingface.co/docs/tokenizers/python/latest/components.html?highlight=unigram#models).
- This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main
- methods. Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should
+ refer to this superclass for more information regarding those methods.
Args:
vocab_file (`str`):
@@ -67,8 +68,8 @@ class ReformerTokenizerFast(PreTrainedTokenizerFast):
- When building a sequence using special tokens, this is not the token that is used for the end of
- sequence. The token used is the `sep_token`.
+ When building a sequence using special tokens, this is not the token that is used for the end of sequence.
+ The token used is the `sep_token`.
diff --git a/src/transformers/models/rembert/configuration_rembert.py b/src/transformers/models/rembert/configuration_rembert.py
index 5459afad4f..b0a4577e0f 100644
--- a/src/transformers/models/rembert/configuration_rembert.py
+++ b/src/transformers/models/rembert/configuration_rembert.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" RemBERT model configuration """
+""" RemBERT model configuration"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging
@@ -28,20 +28,20 @@ REMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class RemBertConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`RemBertModel`]. It is used to
- instantiate an RemBERT model according to the specified arguments, defining the model architecture. Instantiating a
- configuration with the defaults will yield a similar configuration to that of the remert-large architecture.
+ This is the configuration class to store the configuration of a [`RemBertModel`]. It is used to instantiate an
+ RemBERT model according to the specified arguments, defining the model architecture. Instantiating a configuration
+ with the defaults will yield a similar configuration to that of the remert-large architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
vocab_size (`int`, *optional*, defaults to 250300):
Vocabulary size of the RemBERT model. Defines the number of different tokens that can be represented by the
- `inputs_ids` passed when calling [`RemBertModel`] or
- [`TFRemBertModel`]. Vocabulary size of the model. Defines the different tokens that can
- be represented by the *inputs_ids* passed to the forward method of [`RemBertModel`].
+ `inputs_ids` passed when calling [`RemBertModel`] or [`TFRemBertModel`]. Vocabulary size of the model.
+ Defines the different tokens that can be represented by the *inputs_ids* passed to the forward method of
+ [`RemBertModel`].
hidden_size (`int`, *optional*, defaults to 1152):
Dimensionality of the encoder layers and the pooler layer.
num_hidden_layers (`int`, *optional*, defaults to 32):
@@ -55,8 +55,8 @@ class RemBertConfig(PretrainedConfig):
intermediate_size (`int`, *optional*, defaults to 4608):
Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"selu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"selu"` and `"gelu_new"` are supported.
hidden_dropout_prob (`float`, *optional*, defaults to 0):
The dropout probabilitiy for all fully connected layers in the embeddings, encoder, and pooler.
attention_probs_dropout_prob (`float`, *optional*, defaults to 0):
@@ -67,8 +67,7 @@ class RemBertConfig(PretrainedConfig):
The maximum sequence length that this model might ever be used with. Typically set this to something large
just in case (e.g., 512 or 1024 or 2048).
type_vocab_size (`int`, *optional*, defaults to 2):
- The vocabulary size of the `token_type_ids` passed when calling [`RemBertModel`] or
- [`TFRemBertModel`].
+ The vocabulary size of the `token_type_ids` passed when calling [`RemBertModel`] or [`TFRemBertModel`].
initializer_range (`float`, *optional*, defaults to 0.02):
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
layer_norm_eps (`float`, *optional*, defaults to 1e-12):
@@ -83,15 +82,12 @@ class RemBertConfig(PretrainedConfig):
```
- >>> from transformers import RemBertModel, RemBertConfig
- >>> # Initializing a RemBERT rembert style configuration
- >>> configuration = RemBertConfig()
+ >>> from transformers import RemBertModel, RemBertConfig >>> # Initializing a RemBERT rembert style
+ configuration >>> configuration = RemBertConfig()
- >>> # Initializing a model from the rembert style configuration
- >>> model = RemBertModel(configuration)
+ >>> # Initializing a model from the rembert style configuration >>> model = RemBertModel(configuration)
- >>> # Accessing the model configuration
- >>> configuration = model.config
+ >>> # Accessing the model configuration >>> configuration = model.config
"""
model_type = "rembert"
diff --git a/src/transformers/models/rembert/modeling_rembert.py b/src/transformers/models/rembert/modeling_rembert.py
index c6f898322f..5ec67c45de 100755
--- a/src/transformers/models/rembert/modeling_rembert.py
+++ b/src/transformers/models/rembert/modeling_rembert.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" PyTorch RemBERT model. """
+""" PyTorch RemBERT model."""
import math
@@ -683,8 +683,7 @@ REMBERT_START_DOCSTRING = r"""
Parameters:
config ([`RemBertConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
REMBERT_INPUTS_DOCSTRING = r"""
@@ -692,9 +691,8 @@ REMBERT_INPUTS_DOCSTRING = r"""
input_ids (`torch.LongTensor` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`RemBertTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`RemBertTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
@@ -705,14 +703,16 @@ REMBERT_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
@@ -722,9 +722,9 @@ REMBERT_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert *input_ids* indices into associated vectors
- than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -748,10 +748,9 @@ class RemBertModel(RemBertPreTrainedModel):
all you need](https://arxiv.org/abs/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit,
Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin.
- To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration
- set to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder`
- argument and `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an
- input to the forward pass.
+ To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set
+ to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and
+ `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass.
"""
def __init__(self, config, add_pooling_layer=True):
@@ -815,12 +814,12 @@ class RemBertModel(RemBertPreTrainedModel):
- 0 for tokens that are **masked**.
past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ `decoder_input_ids` of shape `(batch_size, sequence_length)`.
use_cache (`bool`, *optional*):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`).
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`).
"""
output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
output_hidden_states = (
@@ -910,7 +909,7 @@ class RemBertModel(RemBertPreTrainedModel):
)
-@add_start_docstrings("""RemBERT Model with a `language modeling` head on top. """, REMBERT_START_DOCSTRING)
+@add_start_docstrings("""RemBERT Model with a `language modeling` head on top.""", REMBERT_START_DOCSTRING)
class RemBertForMaskedLM(RemBertPreTrainedModel):
def __init__(self, config):
super().__init__(config)
@@ -957,8 +956,9 @@ class RemBertForMaskedLM(RemBertPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1011,7 +1011,7 @@ class RemBertForMaskedLM(RemBertPreTrainedModel):
@add_start_docstrings(
- """RemBERT Model with a `language modeling` head on top for CLM fine-tuning. """, REMBERT_START_DOCSTRING
+ """RemBERT Model with a `language modeling` head on top for CLM fine-tuning.""", REMBERT_START_DOCSTRING
)
class RemBertForCausalLM(RemBertPreTrainedModel):
@@ -1066,16 +1066,16 @@ class RemBertForCausalLM(RemBertPreTrainedModel):
- 0 for tokens that are **masked**.
past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ `decoder_input_ids` of shape `(batch_size, sequence_length)`.
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in
`[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are
ignored (masked), the loss is only computed for the tokens with labels n `[0, ..., config.vocab_size]`.
use_cache (`bool`, *optional*):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`).
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`).
Returns:
@@ -1199,8 +1199,9 @@ class RemBertForSequenceClassification(RemBertPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1295,7 +1296,8 @@ class RemBertForMultipleChoice(RemBertPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
+ Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
+ num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
`input_ids` above)
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1477,12 +1479,12 @@ class RemBertForQuestionAnswering(RemBertPreTrainedModel):
r"""
start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the start of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the end of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
diff --git a/src/transformers/models/rembert/modeling_tf_rembert.py b/src/transformers/models/rembert/modeling_tf_rembert.py
index ba29b4bf9f..007cbc2575 100644
--- a/src/transformers/models/rembert/modeling_tf_rembert.py
+++ b/src/transformers/models/rembert/modeling_tf_rembert.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" TF 2.0 RemBERT model. """
+""" TF 2.0 RemBERT model."""
import math
@@ -851,13 +851,13 @@ class TFRemBertPreTrainedModel(TFPreTrainedModel):
REMBERT_START_DOCSTRING = r"""
- This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading or saving, resizing the input
- embeddings, pruning heads etc.)
+ This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use
- it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage
- and behavior.
+ This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it
+ as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and
+ behavior.
@@ -866,11 +866,11 @@ REMBERT_START_DOCSTRING = r"""
- having all inputs as keyword arguments (like PyTorch models), or
- having all inputs as a list, tuple or dict in the first positional arguments.
- This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all
- the tensors in the first argument of the model call function: `model(inputs)`.
+ This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all the
+ tensors in the first argument of the model call function: `model(inputs)`.
- If you choose this second option, there are three possibilities you can use to gather all the input Tensors in
- the first positional argument :
+ If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the
+ first positional argument :
- a single Tensor with `input_ids` only and nothing else: `model(inputs_ids)`
- a list of varying length with one or several input Tensors IN THE ORDER given in the docstring:
@@ -883,8 +883,7 @@ REMBERT_START_DOCSTRING = r"""
Args:
config ([`RemBertConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
REMBERT_INPUTS_DOCSTRING = r"""
@@ -892,9 +891,8 @@ REMBERT_INPUTS_DOCSTRING = r"""
input_ids (`np.ndarray`, `tf.Tensor`, `List[tf.Tensor]` ``Dict[str, tf.Tensor]` or `Dict[str, np.ndarray]` and each example must have the shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`BertTokenizer`]. See
- [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for
- details.
+ Indices can be obtained using [`BertTokenizer`]. See [`PreTrainedTokenizer.__call__`] and
+ [`PreTrainedTokenizer.encode`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`np.ndarray` or `tf.Tensor` of shape `({0})`, *optional*):
@@ -905,14 +903,16 @@ REMBERT_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`np.ndarray` or `tf.Tensor` of shape `({0})`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`np.ndarray` or `tf.Tensor` of shape `({0})`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
head_mask (`np.ndarray` or `tf.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
@@ -922,9 +922,9 @@ REMBERT_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
inputs_embeds (`np.ndarray` or `tf.Tensor` of shape `({0}, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the
@@ -934,8 +934,8 @@ REMBERT_INPUTS_DOCSTRING = r"""
more detail. This argument can be used only in eager mode, in graph mode the value in the config will be
used instead.
return_dict (`bool`, *optional*):
- Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This
- argument can be used in eager mode, in graph mode the value will always be set to True.
+ Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This argument can be used
+ in eager mode, in graph mode the value will always be set to True.
training (`bool`, *optional*, defaults to `False``):
Whether or not to use the model in training mode (some modules like dropout modules have different
behaviors between training and evaluation).
@@ -990,12 +990,12 @@ class TFRemBertModel(TFRemBertPreTrainedModel):
past_key_values (`Tuple[Tuple[tf.Tensor]]` of length `config.n_layers`)
contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ `decoder_input_ids` of shape `(batch_size, sequence_length)`.
use_cache (`bool`, *optional*, defaults to `True`):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`). Set to `False` during training, `True` during generation
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`). Set to `False` during training, `True` during generation
"""
inputs = input_processing(
func=self.call,
@@ -1057,7 +1057,7 @@ class TFRemBertModel(TFRemBertPreTrainedModel):
)
-@add_start_docstrings("""RemBERT Model with a `language modeling` head on top. """, REMBERT_START_DOCSTRING)
+@add_start_docstrings("""RemBERT Model with a `language modeling` head on top.""", REMBERT_START_DOCSTRING)
class TFRemBertForMaskedLM(TFRemBertPreTrainedModel, TFMaskedLanguageModelingLoss):
def __init__(self, config: RemBertConfig, *inputs, **kwargs):
super().__init__(config, *inputs, **kwargs)
@@ -1098,8 +1098,9 @@ class TFRemBertForMaskedLM(TFRemBertPreTrainedModel, TFMaskedLanguageModelingLos
) -> Union[TFMaskedLMOutput, Tuple[tf.Tensor]]:
r"""
labels (`tf.Tensor` or `np.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
"""
inputs = input_processing(
func=self.call,
@@ -1154,7 +1155,7 @@ class TFRemBertForMaskedLM(TFRemBertPreTrainedModel, TFMaskedLanguageModelingLos
@add_start_docstrings(
- """RemBERT Model with a `language modeling` head on top for CLM fine-tuning. """, REMBERT_START_DOCSTRING
+ """RemBERT Model with a `language modeling` head on top for CLM fine-tuning.""", REMBERT_START_DOCSTRING
)
class TFRemBertForCausalLM(TFRemBertPreTrainedModel, TFCausalLanguageModelingLoss):
def __init__(self, config: RemBertConfig, *inputs, **kwargs):
@@ -1220,14 +1221,15 @@ class TFRemBertForCausalLM(TFRemBertPreTrainedModel, TFCausalLanguageModelingLos
past_key_values (`Tuple[Tuple[tf.Tensor]]` of length `config.n_layers`)
contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ `decoder_input_ids` of shape `(batch_size, sequence_length)`.
use_cache (`bool`, *optional*, defaults to `True`):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`). Set to `False` during training, `True` during generation
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`). Set to `False` during training, `True` during generation
labels (`tf.Tensor` or `np.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the cross entropy classification loss. Indices should be in `[0, ..., config.vocab_size - 1]`.
+ Labels for computing the cross entropy classification loss. Indices should be in `[0, ...,
+ config.vocab_size - 1]`.
"""
inputs = input_processing(
func=self.call,
@@ -1347,8 +1349,9 @@ class TFRemBertForSequenceClassification(TFRemBertPreTrainedModel, TFSequenceCla
) -> Union[TFSequenceClassifierOutput, Tuple[tf.Tensor]]:
r"""
labels (`tf.Tensor` or `np.ndarray` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
inputs = input_processing(
func=self.call,
@@ -1452,8 +1455,8 @@ class TFRemBertForMultipleChoice(TFRemBertPreTrainedModel, TFMultipleChoiceLoss)
) -> Union[TFMultipleChoiceModelOutput, Tuple[tf.Tensor]]:
r"""
labels (`tf.Tensor` or `np.ndarray` of shape `(batch_size,)`, *optional*):
- Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices]` where `num_choices` is the size of the second dimension of the input tensors. (See
- `input_ids` above)
+ Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices]`
+ where `num_choices` is the size of the second dimension of the input tensors. (See `input_ids` above)
"""
inputs = input_processing(
func=self.call,
@@ -1692,12 +1695,12 @@ class TFRemBertForQuestionAnswering(TFRemBertPreTrainedModel, TFQuestionAnswerin
r"""
start_positions (`tf.Tensor` or `np.ndarray` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the start of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
end_positions (`tf.Tensor` or `np.ndarray` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the end of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
"""
inputs = input_processing(
func=self.call,
diff --git a/src/transformers/models/rembert/tokenization_rembert.py b/src/transformers/models/rembert/tokenization_rembert.py
index d12cb75a3c..4c2cce94aa 100644
--- a/src/transformers/models/rembert/tokenization_rembert.py
+++ b/src/transformers/models/rembert/tokenization_rembert.py
@@ -44,8 +44,8 @@ class RemBertTokenizer(PreTrainedTokenizer):
"""
Construct a RemBERT tokenizer. Based on [SentencePiece](https://github.com/google/sentencepiece).
- This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods.
- Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
+ this superclass for more information regarding those methods.
Args:
vocab_file (`str`):
@@ -66,8 +66,8 @@ class RemBertTokenizer(PreTrainedTokenizer):
- When building a sequence using special tokens, this is not the token that is used for the end of
- sequence. The token used is the `sep_token`.
+ When building a sequence using special tokens, this is not the token that is used for the end of sequence.
+ The token used is the `sep_token`.
@@ -246,8 +246,7 @@ class RemBertTokenizer(PreTrainedTokenizer):
Optional second list of IDs for sequence pairs.
Returns:
- `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given
- sequence(s).
+ `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
"""
sep = [self.sep_token_id]
cls = [self.cls_token_id]
diff --git a/src/transformers/models/rembert/tokenization_rembert_fast.py b/src/transformers/models/rembert/tokenization_rembert_fast.py
index 335aa92a05..e55df93be6 100644
--- a/src/transformers/models/rembert/tokenization_rembert_fast.py
+++ b/src/transformers/models/rembert/tokenization_rembert_fast.py
@@ -51,9 +51,10 @@ SPIECE_UNDERLINE = "▁"
class RemBertTokenizerFast(PreTrainedTokenizerFast):
"""
- Construct a "fast" RemBert tokenizer (backed by HuggingFace's *tokenizers* library). Based on [Unigram](https://huggingface.co/docs/tokenizers/python/latest/components.html?highlight=unigram#models). This tokenizer
- inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should
- refer to this superclass for more information regarding those methods
+ Construct a "fast" RemBert tokenizer (backed by HuggingFace's *tokenizers* library). Based on
+ [Unigram](https://huggingface.co/docs/tokenizers/python/latest/components.html?highlight=unigram#models). This
+ tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should refer to
+ this superclass for more information regarding those methods
Args:
vocab_file (`str`):
@@ -217,8 +218,7 @@ class RemBertTokenizerFast(PreTrainedTokenizerFast):
Optional second list of IDs for sequence pairs.
Returns:
- `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given
- sequence(s).
+ `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
"""
sep = [self.sep_token_id]
cls = [self.cls_token_id]
diff --git a/src/transformers/models/retribert/configuration_retribert.py b/src/transformers/models/retribert/configuration_retribert.py
index 6f5d15c290..a0fac356d5 100644
--- a/src/transformers/models/retribert/configuration_retribert.py
+++ b/src/transformers/models/retribert/configuration_retribert.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" RetriBERT model configuration """
+""" RetriBERT model configuration"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging
@@ -28,11 +28,11 @@ RETRIBERT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class RetriBertConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`RetriBertModel`]. It is used
- to instantiate a RetriBertModel model according to the specified arguments, defining the model architecture.
+ This is the configuration class to store the configuration of a [`RetriBertModel`]. It is used to instantiate a
+ RetriBertModel model according to the specified arguments, defining the model architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
@@ -48,8 +48,8 @@ class RetriBertConfig(PretrainedConfig):
intermediate_size (`int`, *optional*, defaults to 3072):
Dimensionality of the "intermediate" (often named feed-forward) layer in the Transformer encoder.
hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"silu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"silu"` and `"gelu_new"` are supported.
hidden_dropout_prob (`float`, *optional*, defaults to 0.1):
The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
attention_probs_dropout_prob (`float`, *optional*, defaults to 0.1):
diff --git a/src/transformers/models/retribert/modeling_retribert.py b/src/transformers/models/retribert/modeling_retribert.py
index 8684177951..cae4f8e15a 100644
--- a/src/transformers/models/retribert/modeling_retribert.py
+++ b/src/transformers/models/retribert/modeling_retribert.py
@@ -66,24 +66,23 @@ class RetriBertPreTrainedModel(PreTrainedModel):
RETRIBERT_START_DOCSTRING = r"""
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
Parameters:
config ([`RetriBertConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
@add_start_docstrings(
- """Bert Based model to embed queries or document for document retrieval. """,
+ """Bert Based model to embed queries or document for document retrieval.""",
RETRIBERT_START_DOCSTRING,
)
class RetriBertModel(RetriBertPreTrainedModel):
@@ -182,9 +181,8 @@ class RetriBertModel(RetriBertPreTrainedModel):
input_ids_query (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
Indices of input sequence tokens in the vocabulary for the queries in a batch.
- Indices can be obtained using [`RetriBertTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`]
- for details.
+ Indices can be obtained using [`RetriBertTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask_query (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -200,12 +198,12 @@ class RetriBertModel(RetriBertPreTrainedModel):
Mask to avoid performing attention on documents padding token indices.
checkpoint_batch_size (`int`, *optional*, defaults to ```-1`):
If greater than 0, uses gradient checkpointing to only compute sequence representation on
- `checkpoint_batch_size` examples at a time on the GPU. All query representations are still
- compared to all document representations in the batch.
+ `checkpoint_batch_size` examples at a time on the GPU. All query representations are still compared to
+ all document representations in the batch.
Return:
- `torch.FloatTensor``: The bidirectional cross-entropy loss obtained while trying to match each query to
- its corresponding document and each document to its corresponding query in the batch
+ `torch.FloatTensor``: The bidirectional cross-entropy loss obtained while trying to match each query to its
+ corresponding document and each document to its corresponding query in the batch
"""
device = input_ids_query.device
q_reps = self.embed_questions(input_ids_query, attention_mask_query, checkpoint_batch_size)
diff --git a/src/transformers/models/retribert/tokenization_retribert.py b/src/transformers/models/retribert/tokenization_retribert.py
index 2c0ac65dd2..934054e605 100644
--- a/src/transformers/models/retribert/tokenization_retribert.py
+++ b/src/transformers/models/retribert/tokenization_retribert.py
@@ -42,11 +42,10 @@ class RetriBertTokenizer(BertTokenizer):
r"""
Constructs a RetriBERT tokenizer.
- [`RetroBertTokenizer`] is identical to [`BertTokenizer`] and runs end-to-end
- tokenization: punctuation splitting and wordpiece.
+ [`RetroBertTokenizer`] is identical to [`BertTokenizer`] and runs end-to-end tokenization: punctuation splitting
+ and wordpiece.
- Refer to superclass [`BertTokenizer`] for usage examples and documentation concerning
- parameters.
+ Refer to superclass [`BertTokenizer`] for usage examples and documentation concerning parameters.
"""
vocab_files_names = VOCAB_FILES_NAMES
diff --git a/src/transformers/models/retribert/tokenization_retribert_fast.py b/src/transformers/models/retribert/tokenization_retribert_fast.py
index 71a038bd9e..43cc383721 100644
--- a/src/transformers/models/retribert/tokenization_retribert_fast.py
+++ b/src/transformers/models/retribert/tokenization_retribert_fast.py
@@ -46,11 +46,10 @@ class RetriBertTokenizerFast(BertTokenizerFast):
r"""
Construct a "fast" RetriBERT tokenizer (backed by HuggingFace's *tokenizers* library).
- [`RetriBertTokenizerFast`] is identical to [`BertTokenizerFast`] and runs
- end-to-end tokenization: punctuation splitting and wordpiece.
+ [`RetriBertTokenizerFast`] is identical to [`BertTokenizerFast`] and runs end-to-end tokenization: punctuation
+ splitting and wordpiece.
- Refer to superclass [`BertTokenizerFast`] for usage examples and documentation concerning
- parameters.
+ Refer to superclass [`BertTokenizerFast`] for usage examples and documentation concerning parameters.
"""
vocab_files_names = VOCAB_FILES_NAMES
diff --git a/src/transformers/models/roberta/configuration_roberta.py b/src/transformers/models/roberta/configuration_roberta.py
index dd3697a5f7..6c54cf7ccd 100644
--- a/src/transformers/models/roberta/configuration_roberta.py
+++ b/src/transformers/models/roberta/configuration_roberta.py
@@ -13,7 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" RoBERTa configuration """
+""" RoBERTa configuration"""
from collections import OrderedDict
from typing import Mapping
@@ -36,16 +36,15 @@ ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class RobertaConfig(BertConfig):
r"""
- This is the configuration class to store the configuration of a [`RobertaModel`] or a
- [`TFRobertaModel`]. It is used to instantiate a RoBERTa model according to the specified
- arguments, defining the model architecture.
+ This is the configuration class to store the configuration of a [`RobertaModel`] or a [`TFRobertaModel`]. It is
+ used to instantiate a RoBERTa model according to the specified arguments, defining the model architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
- The [`RobertaConfig`] class directly inherits [`BertConfig`]. It reuses the
- same defaults. Please check the parent class for more information.
+ The [`RobertaConfig`] class directly inherits [`BertConfig`]. It reuses the same defaults. Please check the parent
+ class for more information.
Examples:
diff --git a/src/transformers/models/roberta/modeling_flax_roberta.py b/src/transformers/models/roberta/modeling_flax_roberta.py
index 475edb0b6d..7a33106362 100644
--- a/src/transformers/models/roberta/modeling_flax_roberta.py
+++ b/src/transformers/models/roberta/modeling_flax_roberta.py
@@ -72,12 +72,12 @@ def create_position_ids_from_input_ids(input_ids, padding_idx):
ROBERTA_START_DOCSTRING = r"""
- This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading, saving and converting weights from
- PyTorch models)
+ This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading, saving and converting weights from PyTorch models)
- This model is also a Flax Linen [flax.linen.Module](https://flax.readthedocs.io/en/latest/flax.linen.html#module) subclass. Use it as a regular Flax linen Module
- and refer to the Flax documentation for all matter related to general usage and behavior.
+ This model is also a Flax Linen [flax.linen.Module](https://flax.readthedocs.io/en/latest/flax.linen.html#module)
+ subclass. Use it as a regular Flax linen Module and refer to the Flax documentation for all matter related to
+ general usage and behavior.
Finally, this model supports inherent JAX features such as:
@@ -89,8 +89,7 @@ ROBERTA_START_DOCSTRING = r"""
Parameters:
config ([`RobertaConfig`]): Model configuration class with all the parameters of the
model. Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the
- model weights.
+ configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the model weights.
"""
ROBERTA_INPUTS_DOCSTRING = r"""
@@ -98,9 +97,8 @@ ROBERTA_INPUTS_DOCSTRING = r"""
input_ids (`numpy.ndarray` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`BertTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`BertTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`numpy.ndarray` of shape `({0})`, *optional*):
@@ -111,15 +109,18 @@ ROBERTA_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`numpy.ndarray` of shape `({0})`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`numpy.ndarray` of shape `({0})`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
- head_mask (`numpy.ndarray` of shape `({0})`, `optional): Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`:
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
+ head_mask (`numpy.ndarray` of shape `({0})`, `optional):
+ Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`:
- 1 indicates the head is **not masked**,
- 0 indicates the head is **masked**.
@@ -784,7 +785,7 @@ class FlaxRobertaForMaskedLMModule(nn.Module):
)
-@add_start_docstrings("""RoBERTa Model with a `language modeling` head on top. """, ROBERTA_START_DOCSTRING)
+@add_start_docstrings("""RoBERTa Model with a `language modeling` head on top.""", ROBERTA_START_DOCSTRING)
class FlaxRobertaForMaskedLM(FlaxRobertaPreTrainedModel):
module_class = FlaxRobertaForMaskedLMModule
diff --git a/src/transformers/models/roberta/modeling_roberta.py b/src/transformers/models/roberta/modeling_roberta.py
index b9f3b082f2..bcbe63cc57 100644
--- a/src/transformers/models/roberta/modeling_roberta.py
+++ b/src/transformers/models/roberta/modeling_roberta.py
@@ -13,7 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-"""PyTorch RoBERTa model. """
+"""PyTorch RoBERTa model."""
import math
@@ -624,19 +624,18 @@ class RobertaPreTrainedModel(PreTrainedModel):
ROBERTA_START_DOCSTRING = r"""
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
Parameters:
config ([`RobertaConfig`]): Model configuration class with all the parameters of the
model. Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
ROBERTA_INPUTS_DOCSTRING = r"""
@@ -644,9 +643,8 @@ ROBERTA_INPUTS_DOCSTRING = r"""
input_ids (`torch.LongTensor` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`RobertaTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`RobertaTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
@@ -657,14 +655,16 @@ ROBERTA_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
@@ -674,9 +674,9 @@ ROBERTA_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -700,10 +700,9 @@ class RobertaModel(RobertaPreTrainedModel):
all you need*_ by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz
Kaiser and Illia Polosukhin.
- To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration
- set to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder`
- argument and `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an
- input to the forward pass.
+ To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set
+ to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and
+ `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass.
.. _*Attention is all you need*: https://arxiv.org/abs/1706.03762
@@ -877,7 +876,7 @@ class RobertaModel(RobertaPreTrainedModel):
@add_start_docstrings(
- """RoBERTa Model with a `language modeling` head on top for CLM fine-tuning. """, ROBERTA_START_DOCSTRING
+ """RoBERTa Model with a `language modeling` head on top for CLM fine-tuning.""", ROBERTA_START_DOCSTRING
)
class RobertaForCausalLM(RobertaPreTrainedModel):
_keys_to_ignore_on_save = [r"lm_head.decoder.weight", r"lm_head.decoder.bias"]
@@ -942,12 +941,12 @@ class RobertaForCausalLM(RobertaPreTrainedModel):
past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ `decoder_input_ids` of shape `(batch_size, sequence_length)`.
use_cache (`bool`, *optional*):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`).
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`).
Returns:
@@ -1030,7 +1029,7 @@ class RobertaForCausalLM(RobertaPreTrainedModel):
return reordered_past
-@add_start_docstrings("""RoBERTa Model with a `language modeling` head on top. """, ROBERTA_START_DOCSTRING)
+@add_start_docstrings("""RoBERTa Model with a `language modeling` head on top.""", ROBERTA_START_DOCSTRING)
class RobertaForMaskedLM(RobertaPreTrainedModel):
_keys_to_ignore_on_save = [r"lm_head.decoder.weight", r"lm_head.decoder.bias"]
_keys_to_ignore_on_load_missing = [r"position_ids", r"lm_head.decoder.weight", r"lm_head.decoder.bias"]
@@ -1085,8 +1084,9 @@ class RobertaForMaskedLM(RobertaPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
kwargs (`Dict[str, any]`, optional, defaults to *{}*):
Used to hide legacy arguments that have been deprecated.
"""
@@ -1195,8 +1195,9 @@ class RobertaForSequenceClassification(RobertaPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1291,7 +1292,8 @@ class RobertaForMultipleChoice(RobertaPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
+ Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
+ num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
`input_ids` above)
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1502,12 +1504,12 @@ class RobertaForQuestionAnswering(RobertaPreTrainedModel):
r"""
start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the start of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the end of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
diff --git a/src/transformers/models/roberta/modeling_tf_roberta.py b/src/transformers/models/roberta/modeling_tf_roberta.py
index bc62c636ba..0969538bd0 100644
--- a/src/transformers/models/roberta/modeling_tf_roberta.py
+++ b/src/transformers/models/roberta/modeling_tf_roberta.py
@@ -13,7 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" TF 2.0 RoBERTa model. """
+""" TF 2.0 RoBERTa model."""
import math
import warnings
@@ -829,13 +829,13 @@ class TFRobertaPreTrainedModel(TFPreTrainedModel):
ROBERTA_START_DOCSTRING = r"""
- This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading or saving, resizing the input
- embeddings, pruning heads etc.)
+ This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use
- it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage
- and behavior.
+ This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it
+ as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and
+ behavior.
@@ -844,11 +844,11 @@ ROBERTA_START_DOCSTRING = r"""
- having all inputs as keyword arguments (like PyTorch models), or
- having all inputs as a list, tuple or dict in the first positional arguments.
- This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all
- the tensors in the first argument of the model call function: `model(inputs)`.
+ This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all the
+ tensors in the first argument of the model call function: `model(inputs)`.
- If you choose this second option, there are three possibilities you can use to gather all the input Tensors in
- the first positional argument :
+ If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the
+ first positional argument :
- a single Tensor with `input_ids` only and nothing else: `model(inputs_ids)`
- a list of varying length with one or several input Tensors IN THE ORDER given in the docstring:
@@ -861,8 +861,7 @@ ROBERTA_START_DOCSTRING = r"""
Parameters:
config ([`RobertaConfig`]): Model configuration class with all the parameters of the
model. Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
ROBERTA_INPUTS_DOCSTRING = r"""
@@ -870,9 +869,8 @@ ROBERTA_INPUTS_DOCSTRING = r"""
input_ids (`Numpy array` or `tf.Tensor` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`RobertaTokenizer`]. See
- [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for
- details.
+ Indices can be obtained using [`RobertaTokenizer`]. See [`PreTrainedTokenizer.__call__`] and
+ [`PreTrainedTokenizer.encode`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`Numpy array` or `tf.Tensor` of shape `({0})`, *optional*):
@@ -883,14 +881,16 @@ ROBERTA_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`Numpy array` or `tf.Tensor` of shape `({0})`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`Numpy array` or `tf.Tensor` of shape `({0})`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
head_mask (`Numpy array` or `tf.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
@@ -900,9 +900,9 @@ ROBERTA_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
inputs_embeds (`tf.Tensor` of shape `({0}, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the
@@ -912,8 +912,8 @@ ROBERTA_INPUTS_DOCSTRING = r"""
more detail. This argument can be used only in eager mode, in graph mode the value in the config will be
used instead.
return_dict (`bool`, *optional*):
- Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This
- argument can be used in eager mode, in graph mode the value will always be set to True.
+ Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This argument can be used
+ in eager mode, in graph mode the value will always be set to True.
training (`bool`, *optional*, defaults to `False`):
Whether or not to use the model in training mode (some modules like dropout modules have different
behaviors between training and evaluation).
@@ -967,12 +967,12 @@ class TFRobertaModel(TFRobertaPreTrainedModel):
past_key_values (`Tuple[Tuple[tf.Tensor]]` of length `config.n_layers`)
contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ `decoder_input_ids` of shape `(batch_size, sequence_length)`.
use_cache (`bool`, *optional*, defaults to `True`):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`). Set to `False` during training, `True` during generation
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`). Set to `False` during training, `True` during generation
"""
inputs = input_processing(
func=self.call,
@@ -1086,7 +1086,7 @@ class TFRobertaLMHead(tf.keras.layers.Layer):
return hidden_states
-@add_start_docstrings("""RoBERTa Model with a `language modeling` head on top. """, ROBERTA_START_DOCSTRING)
+@add_start_docstrings("""RoBERTa Model with a `language modeling` head on top.""", ROBERTA_START_DOCSTRING)
class TFRobertaForMaskedLM(TFRobertaPreTrainedModel, TFMaskedLanguageModelingLoss):
# names with a '.' represents the authorized unexpected/missing layers when a TF model is loaded from a PT model
_keys_to_ignore_on_load_unexpected = [r"pooler", r"lm_head.decoder.weight"]
@@ -1128,8 +1128,9 @@ class TFRobertaForMaskedLM(TFRobertaPreTrainedModel, TFMaskedLanguageModelingLos
):
r"""
labels (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
"""
inputs = input_processing(
func=self.call,
@@ -1256,14 +1257,15 @@ class TFRobertaForCausalLM(TFRobertaPreTrainedModel, TFCausalLanguageModelingLos
past_key_values (`Tuple[Tuple[tf.Tensor]]` of length `config.n_layers`)
contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ `decoder_input_ids` of shape `(batch_size, sequence_length)`.
use_cache (`bool`, *optional*, defaults to `True`):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`). Set to `False` during training, `True` during generation
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`). Set to `False` during training, `True` during generation
labels (`tf.Tensor` or `np.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the cross entropy classification loss. Indices should be in `[0, ..., config.vocab_size - 1]`.
+ Labels for computing the cross entropy classification loss. Indices should be in `[0, ...,
+ config.vocab_size - 1]`.
"""
inputs = input_processing(
func=self.call,
@@ -1410,8 +1412,9 @@ class TFRobertaForSequenceClassification(TFRobertaPreTrainedModel, TFSequenceCla
):
r"""
labels (`tf.Tensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
inputs = input_processing(
func=self.call,
@@ -1520,8 +1523,8 @@ class TFRobertaForMultipleChoice(TFRobertaPreTrainedModel, TFMultipleChoiceLoss)
):
r"""
labels (`tf.Tensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices]` where `num_choices` is the size of the second dimension of the input tensors. (See
- `input_ids` above)
+ Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices]`
+ where `num_choices` is the size of the second dimension of the input tensors. (See `input_ids` above)
"""
inputs = input_processing(
func=self.call,
@@ -1759,12 +1762,12 @@ class TFRobertaForQuestionAnswering(TFRobertaPreTrainedModel, TFQuestionAnswerin
r"""
start_positions (`tf.Tensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the start of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
end_positions (`tf.Tensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the end of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
"""
inputs = input_processing(
func=self.call,
diff --git a/src/transformers/models/roberta/tokenization_roberta.py b/src/transformers/models/roberta/tokenization_roberta.py
index 43aa99fc94..f3de26cdee 100644
--- a/src/transformers/models/roberta/tokenization_roberta.py
+++ b/src/transformers/models/roberta/tokenization_roberta.py
@@ -78,13 +78,12 @@ class RobertaTokenizer(GPT2Tokenizer):
- When used with `is_split_into_words=True`, this tokenizer will add a space before each word (even the first
- one).
+ When used with `is_split_into_words=True`, this tokenizer will add a space before each word (even the first one).
- This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods.
- Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
+ this superclass for more information regarding those methods.
Args:
vocab_file (`str`):
@@ -92,7 +91,8 @@ class RobertaTokenizer(GPT2Tokenizer):
merges_file (`str`):
Path to the merges file.
errors (`str`, *optional*, defaults to `"replace"`):
- Paradigm to follow when decoding bytes to UTF-8. See [bytes.decode](https://docs.python.org/3/library/stdtypes.html#bytes.decode) for more information.
+ Paradigm to follow when decoding bytes to UTF-8. See
+ [bytes.decode](https://docs.python.org/3/library/stdtypes.html#bytes.decode) for more information.
bos_token (`str`, *optional*, defaults to `""`):
The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token.
@@ -108,8 +108,8 @@ class RobertaTokenizer(GPT2Tokenizer):
- When building a sequence using special tokens, this is not the token that is used for the end of
- sequence. The token used is the `sep_token`.
+ When building a sequence using special tokens, this is not the token that is used for the end of sequence.
+ The token used is the `sep_token`.
diff --git a/src/transformers/models/roberta/tokenization_roberta_fast.py b/src/transformers/models/roberta/tokenization_roberta_fast.py
index 28162a2994..01b8da730f 100644
--- a/src/transformers/models/roberta/tokenization_roberta_fast.py
+++ b/src/transformers/models/roberta/tokenization_roberta_fast.py
@@ -87,13 +87,12 @@ class RobertaTokenizerFast(GPT2TokenizerFast):
- When used with `is_split_into_words=True`, this tokenizer needs to be instantiated with
- `add_prefix_space=True`.
+ When used with `is_split_into_words=True`, this tokenizer needs to be instantiated with `add_prefix_space=True`.
- This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main
- methods. Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should
+ refer to this superclass for more information regarding those methods.
Args:
vocab_file (`str`):
@@ -101,7 +100,8 @@ class RobertaTokenizerFast(GPT2TokenizerFast):
merges_file (`str`):
Path to the merges file.
errors (`str`, *optional*, defaults to `"replace"`):
- Paradigm to follow when decoding bytes to UTF-8. See [bytes.decode](https://docs.python.org/3/library/stdtypes.html#bytes.decode) for more information.
+ Paradigm to follow when decoding bytes to UTF-8. See
+ [bytes.decode](https://docs.python.org/3/library/stdtypes.html#bytes.decode) for more information.
bos_token (`str`, *optional*, defaults to `""`):
The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token.
@@ -117,8 +117,8 @@ class RobertaTokenizerFast(GPT2TokenizerFast):
- When building a sequence using special tokens, this is not the token that is used for the end of
- sequence. The token used is the `sep_token`.
+ When building a sequence using special tokens, this is not the token that is used for the end of sequence.
+ The token used is the `sep_token`.
@@ -214,8 +214,8 @@ class RobertaTokenizerFast(GPT2TokenizerFast):
@property
def mask_token(self) -> str:
"""
- `str`: Mask token, to use when training a model with masked-language modeling. Log an error if used while
- not having been set.
+ `str`: Mask token, to use when training a model with masked-language modeling. Log an error if used while not
+ having been set.
Roberta tokenizer has a special mask token to be usable in the fill-mask pipeline. The mask token will greedily
comprise the space before the **.
diff --git a/src/transformers/models/roformer/configuration_roformer.py b/src/transformers/models/roformer/configuration_roformer.py
index 9ea3e57a14..bb7961f38e 100644
--- a/src/transformers/models/roformer/configuration_roformer.py
+++ b/src/transformers/models/roformer/configuration_roformer.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" RoFormer model configuration """
+""" RoFormer model configuration"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging
@@ -33,23 +33,21 @@ ROFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class RoFormerConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`RoFormerModel`]. It is used to
- instantiate an RoFormer model according to the specified arguments, defining the model architecture. Instantiating
- a configuration with the defaults will yield a similar configuration to that of the RoFormer
+ This is the configuration class to store the configuration of a [`RoFormerModel`]. It is used to instantiate an
+ RoFormer model according to the specified arguments, defining the model architecture. Instantiating a configuration
+ with the defaults will yield a similar configuration to that of the RoFormer
[junnyu/roformer_chinese_base](https://huggingface.co/junnyu/roformer_chinese_base) architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
vocab_size (`int`, *optional*, defaults to 50000):
Vocabulary size of the RoFormer model. Defines the number of different tokens that can be represented by
- the `inputs_ids` passed when calling [`RoFormerModel`] or
- [`TFRoFormerModel`].
+ the `inputs_ids` passed when calling [`RoFormerModel`] or [`TFRoFormerModel`].
embedding_size (`int`, *optional*, defaults to None):
- Dimensionality of the encoder layers and the pooler layer. Defaults to the `hidden_size` if not
- provided.
+ Dimensionality of the encoder layers and the pooler layer. Defaults to the `hidden_size` if not provided.
hidden_size (`int`, *optional*, defaults to 768):
Dimension of the encoder layers and the pooler layer.
num_hidden_layers (`int`, *optional*, defaults to 12):
@@ -59,8 +57,8 @@ class RoFormerConfig(PretrainedConfig):
intermediate_size (`int`, *optional*, defaults to 3072):
Dimension of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"selu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"selu"` and `"gelu_new"` are supported.
hidden_dropout_prob (`float`, *optional*, defaults to 0.1):
The dropout probabilitiy for all fully connected layers in the embeddings, encoder, and pooler.
attention_probs_dropout_prob (`float`, *optional*, defaults to 0.1):
@@ -69,8 +67,7 @@ class RoFormerConfig(PretrainedConfig):
The maximum sequence length that this model might ever be used with. Typically set this to something large
just in case (e.g., 512 or 1024 or 1536).
type_vocab_size (`int`, *optional*, defaults to 2):
- The vocabulary size of the `token_type_ids` passed when calling [`RoFormerModel`]
- or [`TFRoFormerModel`].
+ The vocabulary size of the `token_type_ids` passed when calling [`RoFormerModel`] or [`TFRoFormerModel`].
initializer_range (`float`, *optional*, defaults to 0.02):
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
layer_norm_eps (`float`, *optional*, defaults to 1e-12):
diff --git a/src/transformers/models/roformer/modeling_roformer.py b/src/transformers/models/roformer/modeling_roformer.py
index 95937e5271..c51145f70a 100644
--- a/src/transformers/models/roformer/modeling_roformer.py
+++ b/src/transformers/models/roformer/modeling_roformer.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" PyTorch RoFormer model. """
+""" PyTorch RoFormer model."""
import math
@@ -739,8 +739,7 @@ ROFORMER_START_DOCSTRING = r"""
Parameters:
config ([`RoFormerConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
ROFORMER_INPUTS_DOCSTRING = r"""
@@ -748,9 +747,8 @@ ROFORMER_INPUTS_DOCSTRING = r"""
input_ids (`torch.LongTensor` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`RoFormerTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`RoFormerTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
@@ -761,7 +759,8 @@ ROFORMER_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
@@ -774,9 +773,9 @@ ROFORMER_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert *input_ids* indices into associated vectors
- than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -800,10 +799,9 @@ class RoFormerModel(RoFormerPreTrainedModel):
all you need](https://arxiv.org/abs/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit,
Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin.
- To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration
- set to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder`
- argument and `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an
- input to the forward pass.
+ To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set
+ to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and
+ `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass.
"""
def __init__(self, config):
@@ -867,12 +865,12 @@ class RoFormerModel(RoFormerPreTrainedModel):
- 0 for tokens that are **masked**.
past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ `decoder_input_ids` of shape `(batch_size, sequence_length)`.
use_cache (`bool`, *optional*):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`).
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`).
"""
output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
output_hidden_states = (
@@ -959,7 +957,7 @@ class RoFormerModel(RoFormerPreTrainedModel):
)
-@add_start_docstrings("""RoFormer Model with a `language modeling` head on top. """, ROFORMER_START_DOCSTRING)
+@add_start_docstrings("""RoFormer Model with a `language modeling` head on top.""", ROFORMER_START_DOCSTRING)
class RoFormerForMaskedLM(RoFormerPreTrainedModel):
def __init__(self, config):
super().__init__(config)
@@ -1005,8 +1003,9 @@ class RoFormerForMaskedLM(RoFormerPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1058,7 +1057,7 @@ class RoFormerForMaskedLM(RoFormerPreTrainedModel):
@add_start_docstrings(
- """RoFormer Model with a `language modeling` head on top for CLM fine-tuning. """, ROFORMER_START_DOCSTRING
+ """RoFormer Model with a `language modeling` head on top for CLM fine-tuning.""", ROFORMER_START_DOCSTRING
)
class RoFormerForCausalLM(RoFormerPreTrainedModel):
@@ -1113,16 +1112,16 @@ class RoFormerForCausalLM(RoFormerPreTrainedModel):
- 0 for tokens that are **masked**.
past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ `decoder_input_ids` of shape `(batch_size, sequence_length)`.
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in
`[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are
ignored (masked), the loss is only computed for the tokens with labels n `[0, ..., config.vocab_size]`.
use_cache (`bool`, *optional*):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`).
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`).
Returns:
@@ -1264,8 +1263,9 @@ class RoFormerForSequenceClassification(RoFormerPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1358,7 +1358,8 @@ class RoFormerForMultipleChoice(RoFormerPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
+ Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
+ num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
`input_ids` above)
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1537,12 +1538,12 @@ class RoFormerForQuestionAnswering(RoFormerPreTrainedModel):
r"""
start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the start of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the end of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
diff --git a/src/transformers/models/roformer/modeling_tf_roformer.py b/src/transformers/models/roformer/modeling_tf_roformer.py
index 66b36e4f70..aadd5012a3 100644
--- a/src/transformers/models/roformer/modeling_tf_roformer.py
+++ b/src/transformers/models/roformer/modeling_tf_roformer.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" TF 2.0 RoFormer model. """
+""" TF 2.0 RoFormer model."""
import math
@@ -715,13 +715,13 @@ class TFRoFormerPreTrainedModel(TFPreTrainedModel):
ROFORMER_START_DOCSTRING = r"""
- This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading or saving, resizing the input
- embeddings, pruning heads etc.)
+ This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use
- it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage
- and behavior.
+ This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it
+ as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and
+ behavior.
@@ -730,11 +730,11 @@ ROFORMER_START_DOCSTRING = r"""
- having all inputs as keyword arguments (like PyTorch models), or
- having all inputs as a list, tuple or dict in the first positional arguments.
- This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all
- the tensors in the first argument of the model call function: `model(inputs)`.
+ This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all the
+ tensors in the first argument of the model call function: `model(inputs)`.
- If you choose this second option, there are three possibilities you can use to gather all the input Tensors in
- the first positional argument :
+ If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the
+ first positional argument :
- a single Tensor with `input_ids` only and nothing else: `model(inputs_ids)`
- a list of varying length with one or several input Tensors IN THE ORDER given in the docstring:
@@ -747,8 +747,7 @@ ROFORMER_START_DOCSTRING = r"""
Args:
config ([`RoFormerConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
ROFORMER_INPUTS_DOCSTRING = r"""
@@ -756,9 +755,8 @@ ROFORMER_INPUTS_DOCSTRING = r"""
input_ids (`np.ndarray`, `tf.Tensor`, `List[tf.Tensor]` ``Dict[str, tf.Tensor]` or `Dict[str, np.ndarray]` and each example must have the shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`RoFormerTokenizer`]. See
- [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for
- details.
+ Indices can be obtained using [`RoFormerTokenizer`]. See [`PreTrainedTokenizer.__call__`] and
+ [`PreTrainedTokenizer.encode`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`np.ndarray` or `tf.Tensor` of shape `({0})`, *optional*):
@@ -769,7 +767,8 @@ ROFORMER_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`np.ndarray` or `tf.Tensor` of shape `({0})`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
@@ -782,9 +781,9 @@ ROFORMER_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
inputs_embeds (`np.ndarray` or `tf.Tensor` of shape `({0}, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the
@@ -794,8 +793,8 @@ ROFORMER_INPUTS_DOCSTRING = r"""
more detail. This argument can be used only in eager mode, in graph mode the value in the config will be
used instead.
return_dict (`bool`, *optional*):
- Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This
- argument can be used in eager mode, in graph mode the value will always be set to True.
+ Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This argument can be used
+ in eager mode, in graph mode the value will always be set to True.
training (`bool`, *optional*, defaults to `False``):
Whether or not to use the model in training mode (some modules like dropout modules have different
behaviors between training and evaluation).
@@ -867,7 +866,7 @@ class TFRoFormerModel(TFRoFormerPreTrainedModel):
return TFBaseModelOutput(last_hidden_state=output.last_hidden_state, hidden_states=hs, attentions=attns)
-@add_start_docstrings("""RoFormer Model with a `language modeling` head on top. """, ROFORMER_START_DOCSTRING)
+@add_start_docstrings("""RoFormer Model with a `language modeling` head on top.""", ROFORMER_START_DOCSTRING)
class TFRoFormerForMaskedLM(TFRoFormerPreTrainedModel, TFMaskedLanguageModelingLoss):
def __init__(self, config: RoFormerConfig, *inputs, **kwargs):
super().__init__(config, *inputs, **kwargs)
@@ -907,8 +906,9 @@ class TFRoFormerForMaskedLM(TFRoFormerPreTrainedModel, TFMaskedLanguageModelingL
) -> Union[TFMaskedLMOutput, Tuple[tf.Tensor]]:
r"""
labels (`tf.Tensor` or `np.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
"""
inputs = input_processing(
func=self.call,
@@ -961,7 +961,7 @@ class TFRoFormerForMaskedLM(TFRoFormerPreTrainedModel, TFMaskedLanguageModelingL
@add_start_docstrings(
- """RoFormer Model with a `language modeling` head on top for CLM fine-tuning. """, ROFORMER_START_DOCSTRING
+ """RoFormer Model with a `language modeling` head on top for CLM fine-tuning.""", ROFORMER_START_DOCSTRING
)
class TFRoFormerForCausalLM(TFRoFormerPreTrainedModel, TFCausalLanguageModelingLoss):
def __init__(self, config: RoFormerConfig, *inputs, **kwargs):
@@ -998,7 +998,8 @@ class TFRoFormerForCausalLM(TFRoFormerPreTrainedModel, TFCausalLanguageModelingL
) -> Union[TFCausalLMOutput, Tuple[tf.Tensor]]:
r"""
labels (`tf.Tensor` or `np.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the cross entropy classification loss. Indices should be in `[0, ..., config.vocab_size - 1]`.
+ Labels for computing the cross entropy classification loss. Indices should be in `[0, ...,
+ config.vocab_size - 1]`.
"""
inputs = input_processing(
func=self.call,
@@ -1122,8 +1123,9 @@ class TFRoFormerForSequenceClassification(TFRoFormerPreTrainedModel, TFSequenceC
) -> Union[TFSequenceClassifierOutput, Tuple[tf.Tensor]]:
r"""
labels (`tf.Tensor` or `np.ndarray` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
inputs = input_processing(
func=self.call,
@@ -1226,8 +1228,8 @@ class TFRoFormerForMultipleChoice(TFRoFormerPreTrainedModel, TFMultipleChoiceLos
) -> Union[TFMultipleChoiceModelOutput, Tuple[tf.Tensor]]:
r"""
labels (`tf.Tensor` or `np.ndarray` of shape `(batch_size,)`, *optional*):
- Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices]` where `num_choices` is the size of the second dimension of the input tensors. (See
- `input_ids` above)
+ Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices]`
+ where `num_choices` is the size of the second dimension of the input tensors. (See `input_ids` above)
"""
inputs = input_processing(
func=self.call,
@@ -1455,12 +1457,12 @@ class TFRoFormerForQuestionAnswering(TFRoFormerPreTrainedModel, TFQuestionAnswer
r"""
start_positions (`tf.Tensor` or `np.ndarray` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the start of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
end_positions (`tf.Tensor` or `np.ndarray` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the end of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
"""
inputs = input_processing(
func=self.call,
diff --git a/src/transformers/models/roformer/tokenization_roformer.py b/src/transformers/models/roformer/tokenization_roformer.py
index 0a5f51a453..e71ac937ca 100644
--- a/src/transformers/models/roformer/tokenization_roformer.py
+++ b/src/transformers/models/roformer/tokenization_roformer.py
@@ -62,8 +62,8 @@ class RoFormerTokenizer(PreTrainedTokenizer):
r"""
Construct a RoFormer tokenizer. Based on [Rust Jieba](https://pypi.org/project/rjieba/).
- This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods.
- Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
+ this superclass for more information regarding those methods.
Args:
vocab_file (`str`):
@@ -93,7 +93,8 @@ class RoFormerTokenizer(PreTrainedTokenizer):
tokenize_chinese_chars (`bool`, *optional*, defaults to `True`):
Whether or not to tokenize Chinese characters.
- This should likely be deactivated for Japanese (see this [issue](https://github.com/huggingface/transformers/issues/328)).
+ This should likely be deactivated for Japanese (see this
+ [issue](https://github.com/huggingface/transformers/issues/328)).
strip_accents: (`bool`, *optional*):
Whether or not to strip all accents. If this option is not specified, then it will be determined by the
value for `lowercase` (as in the original BERT).
@@ -296,8 +297,7 @@ class RoFormerTokenizer(PreTrainedTokenizer):
Optional second list of IDs for sequence pairs.
Returns:
- `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given
- sequence(s).
+ `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
"""
sep = [self.sep_token_id]
cls = [self.cls_token_id]
diff --git a/src/transformers/models/roformer/tokenization_roformer_fast.py b/src/transformers/models/roformer/tokenization_roformer_fast.py
index b7ef87181f..12ccb30afd 100644
--- a/src/transformers/models/roformer/tokenization_roformer_fast.py
+++ b/src/transformers/models/roformer/tokenization_roformer_fast.py
@@ -64,12 +64,11 @@ class RoFormerTokenizerFast(PreTrainedTokenizerFast):
r"""
Construct a "fast" RoFormer tokenizer (backed by HuggingFace's *tokenizers* library).
- [`RoFormerTokenizerFast`] is almost identical to [`BertTokenizerFast`] and
- runs end-to-end tokenization: punctuation splitting and wordpiece. There are some difference between them when
- tokenizing Chinese.
+ [`RoFormerTokenizerFast`] is almost identical to [`BertTokenizerFast`] and runs end-to-end tokenization:
+ punctuation splitting and wordpiece. There are some difference between them when tokenizing Chinese.
- This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main
- methods. Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should
+ refer to this superclass for more information regarding those methods.
Example:
@@ -181,8 +180,7 @@ class RoFormerTokenizerFast(PreTrainedTokenizerFast):
Optional second list of IDs for sequence pairs.
Returns:
- `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given
- sequence(s).
+ `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
"""
sep = [self.sep_token_id]
cls = [self.cls_token_id]
diff --git a/src/transformers/models/segformer/configuration_segformer.py b/src/transformers/models/segformer/configuration_segformer.py
index 750302d1a2..d1790634e6 100644
--- a/src/transformers/models/segformer/configuration_segformer.py
+++ b/src/transformers/models/segformer/configuration_segformer.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" SegFormer model configuration """
+""" SegFormer model configuration"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging
@@ -28,14 +28,14 @@ SEGFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class SegformerConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`SegformerModel`]. It is used
- to instantiate an SegFormer model according to the specified arguments, defining the model architecture.
- Instantiating a configuration with the defaults will yield a similar configuration to that of the SegFormer
+ This is the configuration class to store the configuration of a [`SegformerModel`]. It is used to instantiate an
+ SegFormer model according to the specified arguments, defining the model architecture. Instantiating a
+ configuration with the defaults will yield a similar configuration to that of the SegFormer
[nvidia/segformer-b0-finetuned-ade-512-512](https://huggingface.co/nvidia/segformer-b0-finetuned-ade-512-512)
architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
image_size (`int`, *optional*, defaults to 512):
@@ -62,8 +62,8 @@ class SegformerConfig(PretrainedConfig):
Ratio of the size of the hidden layer compared to the size of the input layer of the Mix FFNs in the
encoder blocks.
hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"selu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"selu"` and `"gelu_new"` are supported.
hidden_dropout_prob (`float`, *optional*, defaults to 0.0):
The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
attention_probs_dropout_prob (`float`, *optional*, defaults to 0.0):
@@ -79,8 +79,8 @@ class SegformerConfig(PretrainedConfig):
decoder_hidden_size (`int`, *optional*, defaults to 256):
The dimension of the all-MLP decode head.
reshape_last_stage (`bool`, *optional*, defaults to `True`):
- Whether to reshape the features of the last stage back to `(batch_size, num_channels, height, width)`.
- Only required for the semantic segmentation model.
+ Whether to reshape the features of the last stage back to `(batch_size, num_channels, height, width)`. Only
+ required for the semantic segmentation model.
semantic_loss_ignore_index (`int`, *optional*, defaults to 255):
The index that is ignored by the loss function of the semantic segmentation model.
diff --git a/src/transformers/models/segformer/feature_extraction_segformer.py b/src/transformers/models/segformer/feature_extraction_segformer.py
index 14c6619446..cfc53f2be9 100644
--- a/src/transformers/models/segformer/feature_extraction_segformer.py
+++ b/src/transformers/models/segformer/feature_extraction_segformer.py
@@ -38,20 +38,20 @@ class SegformerFeatureExtractor(FeatureExtractionMixin, ImageFeatureExtractionMi
r"""
Constructs a SegFormer feature extractor.
- This feature extractor inherits from [`FeatureExtractionMixin`] which contains most of the main
- methods. Users should refer to this superclass for more information regarding those methods.
+ This feature extractor inherits from [`FeatureExtractionMixin`] which contains most of the main methods. Users
+ should refer to this superclass for more information regarding those methods.
Args:
do_resize (`bool`, *optional*, defaults to `True`):
Whether to resize the input based on a certain `size`.
size (`int` or `Tuple(int)`, *optional*, defaults to 512):
Resize the input to the given size. If a tuple is provided, it should be (width, height). If only an
- integer is provided, then the input will be resized to (size, size). Only has an effect if `do_resize`
- is set to `True`.
+ integer is provided, then the input will be resized to (size, size). Only has an effect if `do_resize` is
+ set to `True`.
resample (`int`, *optional*, defaults to `PIL.Image.BILINEAR`):
An optional resampling filter. This can be one of `PIL.Image.NEAREST`, `PIL.Image.BOX`,
- `PIL.Image.BILINEAR`, `PIL.Image.HAMMING`, `PIL.Image.BICUBIC` or `PIL.Image.LANCZOS`.
- Only has an effect if `do_resize` is set to `True`.
+ `PIL.Image.BILINEAR`, `PIL.Image.HAMMING`, `PIL.Image.BICUBIC` or `PIL.Image.LANCZOS`. Only has an effect
+ if `do_resize` is set to `True`.
do_normalize (`bool`, *optional*, defaults to `True`):
Whether or not to normalize the input with mean and standard deviation.
image_mean (`int`, *optional*, defaults to `[0.485, 0.456, 0.406]`):
diff --git a/src/transformers/models/segformer/modeling_segformer.py b/src/transformers/models/segformer/modeling_segformer.py
index 51313688f6..4dd4ce7609 100755
--- a/src/transformers/models/segformer/modeling_segformer.py
+++ b/src/transformers/models/segformer/modeling_segformer.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" PyTorch SegFormer model. """
+""" PyTorch SegFormer model."""
import collections
@@ -433,8 +433,7 @@ SEGFORMER_START_DOCSTRING = r"""
Parameters:
config ([`SegformerConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
SEGFORMER_INPUTS_DOCSTRING = r"""
@@ -442,8 +441,7 @@ SEGFORMER_INPUTS_DOCSTRING = r"""
Args:
pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using
- [`SegformerFeatureExtractor`]. See
- [`SegformerFeatureExtractor.__call__`] for details.
+ [`SegformerFeatureExtractor`]. See [`SegformerFeatureExtractor.__call__`] for details.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
@@ -559,8 +557,9 @@ class SegformerForImageClassification(SegformerPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the image classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
Returns:
@@ -691,7 +690,7 @@ class SegformerDecodeHead(SegformerPreTrainedModel):
@add_start_docstrings(
- """SegFormer Model transformer with an all-MLP decode head on top e.g. for ADE20k, CityScapes. """,
+ """SegFormer Model transformer with an all-MLP decode head on top e.g. for ADE20k, CityScapes.""",
SEGFORMER_START_DOCSTRING,
)
class SegformerForSemanticSegmentation(SegformerPreTrainedModel):
@@ -715,8 +714,8 @@ class SegformerForSemanticSegmentation(SegformerPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size, height, width)`, *optional*):
- Ground truth semantic segmentation maps for computing the loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels > 1`, a classification loss is computed
- (Cross-Entropy).
+ Ground truth semantic segmentation maps for computing the loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels > 1`, a classification loss is computed (Cross-Entropy).
Returns:
diff --git a/src/transformers/models/sew/configuration_sew.py b/src/transformers/models/sew/configuration_sew.py
index cd939e9d71..9b05e23aa9 100644
--- a/src/transformers/models/sew/configuration_sew.py
+++ b/src/transformers/models/sew/configuration_sew.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" SEW model configuration """
+""" SEW model configuration"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging
@@ -28,12 +28,13 @@ SEW_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class SEWConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`SEWModel`]. It is used to
- instantiate a SEW model according to the specified arguments, defining the model architecture. Instantiating a
- configuration with the defaults will yield a similar configuration to that of the SEW [asapp/sew-tiny-100k](https://huggingface.co/asapp/sew-tiny-100k) architecture.
+ This is the configuration class to store the configuration of a [`SEWModel`]. It is used to instantiate a SEW model
+ according to the specified arguments, defining the model architecture. Instantiating a configuration with the
+ defaults will yield a similar configuration to that of the SEW
+ [asapp/sew-tiny-100k](https://huggingface.co/asapp/sew-tiny-100k) architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
@@ -51,8 +52,8 @@ class SEWConfig(PretrainedConfig):
squeeze_factor (`int`, *optional*, defaults to 2):
Sequence length downsampling factor after the encoder and upsampling factor after the transformer.
hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"selu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"selu"` and `"gelu_new"` are supported.
hidden_dropout (`float`, *optional*, defaults to 0.1):
The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
attention_dropout (`float`, *optional*, defaults to 0.1):
@@ -91,25 +92,27 @@ class SEWConfig(PretrainedConfig):
Number of groups of 1D convolutional positional embeddings layer.
apply_spec_augment (`bool`, *optional*, defaults to `True`):
Whether to apply *SpecAugment* data augmentation to the outputs of the feature extractor. For reference see
- [SpecAugment: A Simple Data Augmentation Method for Automatic Speech Recognition](https://arxiv.org/abs/1904.08779).
+ [SpecAugment: A Simple Data Augmentation Method for Automatic Speech
+ Recognition](https://arxiv.org/abs/1904.08779).
mask_time_prob (`float`, *optional*, defaults to 0.05):
Percentage (between 0 and 1) of all feature vectors along the time axis which will be masked. The masking
procecure generates ''mask_time_prob*len(time_axis)/mask_time_length'' independent masks over the axis. If
reasoning from the propability of each feature vector to be chosen as the start of the vector span to be
- masked, *mask_time_prob* should be `prob_vector_start*mask_time_length`. Note that overlap may decrease
- the actual percentage of masked vectors. This is only relevant if `apply_spec_augment is True`.
+ masked, *mask_time_prob* should be `prob_vector_start*mask_time_length`. Note that overlap may decrease the
+ actual percentage of masked vectors. This is only relevant if `apply_spec_augment is True`.
mask_time_length (`int`, *optional*, defaults to 10):
Length of vector span along the time axis.
mask_time_min_masks (`int`, *optional*, defaults to 2),:
- The minimum number of masks of length `mask_feature_length` generated along the time axis, each time
- step, irrespectively of `mask_feature_prob`. Only relevant if
- ''mask_time_prob*len(time_axis)/mask_time_length < mask_time_min_masks''
+ The minimum number of masks of length `mask_feature_length` generated along the time axis, each time step,
+ irrespectively of `mask_feature_prob`. Only relevant if ''mask_time_prob*len(time_axis)/mask_time_length <
+ mask_time_min_masks''
mask_feature_prob (`float`, *optional*, defaults to 0.0):
Percentage (between 0 and 1) of all feature vectors along the feature axis which will be masked. The
masking procecure generates ''mask_feature_prob*len(feature_axis)/mask_time_length'' independent masks over
the axis. If reasoning from the propability of each feature vector to be chosen as the start of the vector
- span to be masked, *mask_feature_prob* should be `prob_vector_start*mask_feature_length`. Note that
- overlap may decrease the actual percentage of masked vectors. This is only relevant if `apply_spec_augment is True`.
+ span to be masked, *mask_feature_prob* should be `prob_vector_start*mask_feature_length`. Note that overlap
+ may decrease the actual percentage of masked vectors. This is only relevant if `apply_spec_augment is
+ True`.
mask_feature_length (`int`, *optional*, defaults to 10):
Length of vector span along the feature axis.
mask_feature_min_masks (`int`, *optional*, defaults to 0),:
@@ -120,9 +123,9 @@ class SEWConfig(PretrainedConfig):
Specifies the reduction to apply to the output of `torch.nn.CTCLoss`. Only relevant when training an
instance of [`SEWForCTC`].
ctc_zero_infinity (`bool`, *optional*, defaults to `False`):
- Whether to zero infinite losses and the associated gradients of `torch.nn.CTCLoss`. Infinite losses
- mainly occur when the inputs are too short to be aligned to the targets. Only relevant when training an
- instance of [`SEWForCTC`].
+ Whether to zero infinite losses and the associated gradients of `torch.nn.CTCLoss`. Infinite losses mainly
+ occur when the inputs are too short to be aligned to the targets. Only relevant when training an instance
+ of [`SEWForCTC`].
use_weighted_layer_sum (`bool`, *optional*, defaults to `False`):
Whether to use a weighted average of layer outputs with learned weights. Only relevant when using an
instance of [`Wav2Vec2ForSequenceClassification`].
diff --git a/src/transformers/models/sew/modeling_sew.py b/src/transformers/models/sew/modeling_sew.py
index 3f754b8512..7a467f51df 100644
--- a/src/transformers/models/sew/modeling_sew.py
+++ b/src/transformers/models/sew/modeling_sew.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" PyTorch SEW model. """
+""" PyTorch SEW model."""
import math
from typing import Optional, Tuple, Union
@@ -63,8 +63,8 @@ def _compute_mask_indices(
) -> np.ndarray:
"""
Computes random mask spans for a given shape. Used to implement [SpecAugment: A Simple Data Augmentation Method for
- ASR](https://arxiv.org/abs/1904.08779). Note that this method is not optimized to run on TPU and should be run
- on CPU as part of the preprocessing during training.
+ ASR](https://arxiv.org/abs/1904.08779). Note that this method is not optimized to run on TPU and should be run on
+ CPU as part of the preprocessing during training.
Args:
shape: The shape for which to compute masks. This should be of a tuple of size 2 where
@@ -744,11 +744,12 @@ class SEWPreTrainedModel(PreTrainedModel):
SEW_START_DOCSTRING = r"""
- SEW was proposed in [Performance-Efficiency Trade-offs in Unsupervised Pre-training for Speech Recognition](https://arxiv.org/abs/2109.06870) by Felix Wu, Kwangyoun Kim, Jing Pan, Kyu Han, Kilian Q. Weinberger, Yoav
- Artzi.
+ SEW was proposed in [Performance-Efficiency Trade-offs in Unsupervised Pre-training for Speech
+ Recognition](https://arxiv.org/abs/2109.06870) by Felix Wu, Kwangyoun Kim, Jing Pan, Kyu Han, Kilian Q. Weinberger,
+ Yoav Artzi.
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving etc.).
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving etc.).
This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use
it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
@@ -757,8 +758,7 @@ SEW_START_DOCSTRING = r"""
Parameters:
config ([`SEWConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
@@ -767,11 +767,11 @@ SEW_INPUTS_DOCSTRING = r"""
input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`):
Float values of input raw speech waveform. Values can be obtained by loading a *.flac* or *.wav* audio file
into an array of type *List[float]* or a *numpy.ndarray*, *e.g.* via the soundfile library (*pip install
- soundfile*). To prepare the array into *input_values*, the [`Wav2Vec2Processor`] should
- be used for padding and conversion into a tensor of type *torch.FloatTensor*. See
- [`Wav2Vec2Processor.__call__`] for details.
+ soundfile*). To prepare the array into *input_values*, the [`Wav2Vec2Processor`] should be used for padding
+ and conversion into a tensor of type *torch.FloatTensor*. See [`Wav2Vec2Processor.__call__`] for details.
attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0, 1]`:
+ Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0,
+ 1]`:
- 1 for tokens that are **not masked**,
- 0 for tokens that are **masked**.
@@ -917,7 +917,7 @@ class SEWModel(SEWPreTrainedModel):
@add_start_docstrings(
- """SEW Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC). """,
+ """SEW Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC).""",
SEW_START_DOCSTRING,
)
# Copied from transformers.models.wav2vec2.modeling_wav2vec2.Wav2Vec2ForCTC with Wav2Vec2->SEW, wav2vec2->sew, WAV_2_VEC_2->SEW
@@ -966,7 +966,9 @@ class SEWForCTC(SEWPreTrainedModel):
r"""
labels (`torch.LongTensor` of shape `(batch_size, target_length)`, *optional*):
Labels for connectionist temporal classification. Note that `target_length` has to be smaller or equal to
- the sequence length of the output logits. Indices are selected in `[-100, 0, ..., config.vocab_size - 1]`. All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size - 1]`.
+ the sequence length of the output logits. Indices are selected in `[-100, 0, ..., config.vocab_size - 1]`.
+ All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ...,
+ config.vocab_size - 1]`.
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1081,8 +1083,9 @@ class SEWForSequenceClassification(SEWPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
diff --git a/src/transformers/models/sew_d/configuration_sew_d.py b/src/transformers/models/sew_d/configuration_sew_d.py
index 31727e85a5..eb5d9d99c1 100644
--- a/src/transformers/models/sew_d/configuration_sew_d.py
+++ b/src/transformers/models/sew_d/configuration_sew_d.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" SEW-D model configuration """
+""" SEW-D model configuration"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging
@@ -28,12 +28,13 @@ SEW_D_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class SEWDConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`SEWDModel`]. It is used to
- instantiate a SEW-D model according to the specified arguments, defining the model architecture. Instantiating a
- configuration with the defaults will yield a similar configuration to that of the SEW-D [asapp/sew-d-tiny-100k](https://huggingface.co/asapp/sew-d-tiny-100k) architecture.
+ This is the configuration class to store the configuration of a [`SEWDModel`]. It is used to instantiate a SEW-D
+ model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
+ defaults will yield a similar configuration to that of the SEW-D
+ [asapp/sew-d-tiny-100k](https://huggingface.co/asapp/sew-d-tiny-100k) architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
@@ -67,8 +68,8 @@ class SEWDConfig(PretrainedConfig):
norm_rel_ebd (`str`, *optional*, defaults to `"layer_norm"`):
Whether to use layer norm in relative embedding (`"layer_norm"` if yes)
hidden_act (`str` or `function`, *optional*, defaults to `"gelu_python"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"selu"`, `"gelu_python"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"selu"`, `"gelu_python"` and `"gelu_new"` are supported.
hidden_dropout (`float`, *optional*, defaults to 0.1):
The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
attention_dropout (`float`, *optional*, defaults to 0.1):
@@ -109,25 +110,27 @@ class SEWDConfig(PretrainedConfig):
Number of groups of 1D convolutional positional embeddings layer.
apply_spec_augment (`bool`, *optional*, defaults to `True`):
Whether to apply *SpecAugment* data augmentation to the outputs of the feature extractor. For reference see
- [SpecAugment: A Simple Data Augmentation Method for Automatic Speech Recognition](https://arxiv.org/abs/1904.08779).
+ [SpecAugment: A Simple Data Augmentation Method for Automatic Speech
+ Recognition](https://arxiv.org/abs/1904.08779).
mask_time_prob (`float`, *optional*, defaults to 0.05):
Percentage (between 0 and 1) of all feature vectors along the time axis which will be masked. The masking
procecure generates ''mask_time_prob*len(time_axis)/mask_time_length'' independent masks over the axis. If
reasoning from the propability of each feature vector to be chosen as the start of the vector span to be
- masked, *mask_time_prob* should be `prob_vector_start*mask_time_length`. Note that overlap may decrease
- the actual percentage of masked vectors. This is only relevant if `apply_spec_augment is True`.
+ masked, *mask_time_prob* should be `prob_vector_start*mask_time_length`. Note that overlap may decrease the
+ actual percentage of masked vectors. This is only relevant if `apply_spec_augment is True`.
mask_time_length (`int`, *optional*, defaults to 10):
Length of vector span along the time axis.
mask_time_min_masks (`int`, *optional*, defaults to 2),:
- The minimum number of masks of length `mask_feature_length` generated along the time axis, each time
- step, irrespectively of `mask_feature_prob`. Only relevant if
- ''mask_time_prob*len(time_axis)/mask_time_length < mask_time_min_masks''
+ The minimum number of masks of length `mask_feature_length` generated along the time axis, each time step,
+ irrespectively of `mask_feature_prob`. Only relevant if ''mask_time_prob*len(time_axis)/mask_time_length <
+ mask_time_min_masks''
mask_feature_prob (`float`, *optional*, defaults to 0.0):
Percentage (between 0 and 1) of all feature vectors along the feature axis which will be masked. The
masking procecure generates ''mask_feature_prob*len(feature_axis)/mask_time_length'' independent masks over
the axis. If reasoning from the propability of each feature vector to be chosen as the start of the vector
- span to be masked, *mask_feature_prob* should be `prob_vector_start*mask_feature_length`. Note that
- overlap may decrease the actual percentage of masked vectors. This is only relevant if `apply_spec_augment is True`.
+ span to be masked, *mask_feature_prob* should be `prob_vector_start*mask_feature_length`. Note that overlap
+ may decrease the actual percentage of masked vectors. This is only relevant if `apply_spec_augment is
+ True`.
mask_feature_length (`int`, *optional*, defaults to 10):
Length of vector span along the feature axis.
mask_feature_min_masks (`int`, *optional*, defaults to 0),:
@@ -140,9 +143,9 @@ class SEWDConfig(PretrainedConfig):
Specifies the reduction to apply to the output of `torch.nn.CTCLoss`. Only relevant when training an
instance of [`SEWDForCTC`].
ctc_zero_infinity (`bool`, *optional*, defaults to `False`):
- Whether to zero infinite losses and the associated gradients of `torch.nn.CTCLoss`. Infinite losses
- mainly occur when the inputs are too short to be aligned to the targets. Only relevant when training an
- instance of [`SEWDForCTC`].
+ Whether to zero infinite losses and the associated gradients of `torch.nn.CTCLoss`. Infinite losses mainly
+ occur when the inputs are too short to be aligned to the targets. Only relevant when training an instance
+ of [`SEWDForCTC`].
use_weighted_layer_sum (`bool`, *optional*, defaults to `False`):
Whether to use a weighted average of layer outputs with learned weights. Only relevant when using an
instance of [`Wav2Vec2ForSequenceClassification`].
diff --git a/src/transformers/models/sew_d/modeling_sew_d.py b/src/transformers/models/sew_d/modeling_sew_d.py
index be3ade458d..391b3c5b3c 100644
--- a/src/transformers/models/sew_d/modeling_sew_d.py
+++ b/src/transformers/models/sew_d/modeling_sew_d.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" PyTorch SEW model. """
+""" PyTorch SEW model."""
import math
from collections.abc import Sequence
@@ -69,8 +69,8 @@ def _compute_mask_indices(
) -> np.ndarray:
"""
Computes random mask spans for a given shape. Used to implement [SpecAugment: A Simple Data Augmentation Method for
- ASR](https://arxiv.org/abs/1904.08779). Note that this method is not optimized to run on TPU and should be run
- on CPU as part of the preprocessing during training.
+ ASR](https://arxiv.org/abs/1904.08779). Note that this method is not optimized to run on TPU and should be run on
+ CPU as part of the preprocessing during training.
Args:
shape: The shape for which to compute masks. This should be of a tuple of size 2 where
@@ -181,8 +181,8 @@ def build_relative_position(query_size, key_size, bucket_size=-1, max_position=-
Build relative position according to the query and key
We assume the absolute position of query \\(P_q\\) is range from (0, query_size) and the absolute position of key
- \\(P_k\\) is range from (0, key_size), The relative positions from query to key is \\(R_{q \\rightarrow k} =
- P_q - P_k\\)
+ \\(P_k\\) is range from (0, key_size), The relative positions from query to key is \\(R_{q \\rightarrow k} = P_q -
+ P_k\\)
Args:
query_size (int): the length of query
@@ -469,7 +469,8 @@ class XSoftmax(torch.autograd.Function):
Args:
input (`torch.tensor`): The input tensor that will apply softmax.
- mask (`torch.IntTensor`): The mask matrix where 0 indicate that element will be ignored in the softmax calculation.
+ mask (`torch.IntTensor`):
+ The mask matrix where 0 indicate that element will be ignored in the softmax calculation.
dim (int): The dimension that will apply softmax
Example:
@@ -1278,11 +1279,12 @@ class SEWDPreTrainedModel(PreTrainedModel):
SEWD_START_DOCSTRING = r"""
- SEW-D was proposed in [Performance-Efficiency Trade-offs in Unsupervised Pre-training for Speech Recognition](https://arxiv.org/abs/2109.06870) by Felix Wu, Kwangyoun Kim, Jing Pan, Kyu Han, Kilian Q. Weinberger, Yoav
- Artzi.
+ SEW-D was proposed in [Performance-Efficiency Trade-offs in Unsupervised Pre-training for Speech
+ Recognition](https://arxiv.org/abs/2109.06870) by Felix Wu, Kwangyoun Kim, Jing Pan, Kyu Han, Kilian Q. Weinberger,
+ Yoav Artzi.
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving etc.).
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving etc.).
This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use
it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
@@ -1291,8 +1293,7 @@ SEWD_START_DOCSTRING = r"""
Parameters:
config ([`SEWDConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
@@ -1301,11 +1302,11 @@ SEWD_INPUTS_DOCSTRING = r"""
input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`):
Float values of input raw speech waveform. Values can be obtained by loading a *.flac* or *.wav* audio file
into an array of type *List[float]* or a *numpy.ndarray*, *e.g.* via the soundfile library (*pip install
- soundfile*). To prepare the array into *input_values*, the [`Wav2Vec2Processor`] should
- be used for padding and conversion into a tensor of type *torch.FloatTensor*. See
- [`Wav2Vec2Processor.__call__`] for details.
+ soundfile*). To prepare the array into *input_values*, the [`Wav2Vec2Processor`] should be used for padding
+ and conversion into a tensor of type *torch.FloatTensor*. See [`Wav2Vec2Processor.__call__`] for details.
attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0, 1]`:
+ Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0,
+ 1]`:
- 1 for tokens that are **not masked**,
- 0 for tokens that are **masked**.
@@ -1452,7 +1453,7 @@ class SEWDModel(SEWDPreTrainedModel):
@add_start_docstrings(
- """SEW-D Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC). """,
+ """SEW-D Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC).""",
SEWD_START_DOCSTRING,
)
# Copied from transformers.models.wav2vec2.modeling_wav2vec2.Wav2Vec2ForCTC with Wav2Vec2->SEWD, wav2vec2->sew_d, WAV_2_VEC_2->SEWD
@@ -1501,7 +1502,9 @@ class SEWDForCTC(SEWDPreTrainedModel):
r"""
labels (`torch.LongTensor` of shape `(batch_size, target_length)`, *optional*):
Labels for connectionist temporal classification. Note that `target_length` has to be smaller or equal to
- the sequence length of the output logits. Indices are selected in `[-100, 0, ..., config.vocab_size - 1]`. All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size - 1]`.
+ the sequence length of the output logits. Indices are selected in `[-100, 0, ..., config.vocab_size - 1]`.
+ All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ...,
+ config.vocab_size - 1]`.
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1616,8 +1619,9 @@ class SEWDForSequenceClassification(SEWDPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
diff --git a/src/transformers/models/speech_encoder_decoder/configuration_speech_encoder_decoder.py b/src/transformers/models/speech_encoder_decoder/configuration_speech_encoder_decoder.py
index 6fa18bc124..c7d992c076 100644
--- a/src/transformers/models/speech_encoder_decoder/configuration_speech_encoder_decoder.py
+++ b/src/transformers/models/speech_encoder_decoder/configuration_speech_encoder_decoder.py
@@ -27,20 +27,20 @@ logger = logging.get_logger(__name__)
class SpeechEncoderDecoderConfig(PretrainedConfig):
r"""
[`SpeechEncoderDecoderConfig`] is the configuration class to store the configuration of a
- [`SpeechEncoderDecoderModel`]. It is used to instantiate an Encoder Decoder model according to
- the specified arguments, defining the encoder and decoder configs.
+ [`SpeechEncoderDecoderModel`]. It is used to instantiate an Encoder Decoder model according to the specified
+ arguments, defining the encoder and decoder configs.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
kwargs (*optional*):
Dictionary of keyword arguments. Notably:
- - **encoder** ([`PretrainedConfig`], *optional*) -- An instance of a configuration
- object that defines the encoder config.
- - **decoder** ([`PretrainedConfig`], *optional*) -- An instance of a configuration
- object that defines the decoder config.
+ - **encoder** ([`PretrainedConfig`], *optional*) -- An instance of a configuration object that defines
+ the encoder config.
+ - **decoder** ([`PretrainedConfig`], *optional*) -- An instance of a configuration object that defines
+ the decoder config.
Examples:
@@ -94,8 +94,8 @@ class SpeechEncoderDecoderConfig(PretrainedConfig):
cls, encoder_config: PretrainedConfig, decoder_config: PretrainedConfig, **kwargs
) -> PretrainedConfig:
r"""
- Instantiate a [`SpeechEncoderDecoderConfig`] (or a derived class) from a pre-trained encoder
- model configuration and decoder model configuration.
+ Instantiate a [`SpeechEncoderDecoderConfig`] (or a derived class) from a pre-trained encoder model
+ configuration and decoder model configuration.
Returns:
[`SpeechEncoderDecoderConfig`]: An instance of a configuration object
diff --git a/src/transformers/models/speech_encoder_decoder/modeling_speech_encoder_decoder.py b/src/transformers/models/speech_encoder_decoder/modeling_speech_encoder_decoder.py
index cc5ca40f06..dbd80963a8 100644
--- a/src/transformers/models/speech_encoder_decoder/modeling_speech_encoder_decoder.py
+++ b/src/transformers/models/speech_encoder_decoder/modeling_speech_encoder_decoder.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" Classes to support Speech-Encoder-Text-Decoder architectures """
+""" Classes to support Speech-Encoder-Text-Decoder architectures"""
from typing import Optional
@@ -39,41 +39,43 @@ SPEECH_ENCODER_DECODER_START_DOCSTRING = r"""
This class can be used to initialize a speech-sequence-to-text-sequence model with any pretrained speech
autoencoding model as the encoder and any pretrained text autoregressive model as the decoder. The encoder is
loaded via [`~AutoModel.from_pretrained`] function and the decoder is loaded via
- [`~AutoModelForCausalLM.from_pretrained`] function. Cross-attention layers are automatically added
- to the decoder and should be fine-tuned on a downstream generative task, like summarization.
+ [`~AutoModelForCausalLM.from_pretrained`] function. Cross-attention layers are automatically added to the decoder
+ and should be fine-tuned on a downstream generative task, like summarization.
The effectiveness of initializing sequence-to-sequence models with pretrained checkpoints for sequence generation
- tasks was shown in [Leveraging Pre-trained Checkpoints for Sequence Generation Tasks](https://arxiv.org/abs/1907.12461) by Sascha Rothe, Shashi Narayan, Aliaksei Severyn. Michael Matena, Yanqi
+ tasks was shown in [Leveraging Pre-trained Checkpoints for Sequence Generation
+ Tasks](https://arxiv.org/abs/1907.12461) by Sascha Rothe, Shashi Narayan, Aliaksei Severyn. Michael Matena, Yanqi
Zhou, Wei Li, Peter J. Liu.
- Additionally, in [Large-Scale Self- and Semi-Supervised Learning for Speech Translation](https://arxiv.org/abs/2104.06678) it is shown how leveraging large pretrained speech models for speech
+ Additionally, in [Large-Scale Self- and Semi-Supervised Learning for Speech
+ Translation](https://arxiv.org/abs/2104.06678) it is shown how leveraging large pretrained speech models for speech
translation yields a significant performance improvement.
After such an Speech-Encoder Decoder model has been trained/fine-tuned, it can be saved/loaded just like any other
models (see the examples for more information).
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
Parameters:
config ([`SpeechEncoderDecoderConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
SPEECH_ENCODER_DECODER_INPUTS_DOCSTRING = r"""
Args:
inputs (`torch.FloatTensor` of shape `(batch_size, sequence_length)` or `(batch_size, sequence_length, feature_dim)`, *optional*):
- Float values of input raw speech waveform or speech features. Values can be obtained by loading a *.flac* or *.wav* audio file
- into an array of type *List[float]* or a *numpy.ndarray*, *e.g.* via the soundfile library (*pip install
- soundfile*). To prepare the array into *inputs*, either the [`Wav2Vec2Processor`] or [`Speech2TextProcessor`] should
- be used for padding and conversion into a tensor of type *torch.FloatTensor*.
+ Float values of input raw speech waveform or speech features. Values can be obtained by loading a *.flac*
+ or *.wav* audio file into an array of type *List[float]* or a *numpy.ndarray*, *e.g.* via the soundfile
+ library (*pip install soundfile*). To prepare the array into *inputs*, either the [`Wav2Vec2Processor`] or
+ [`Speech2TextProcessor`] should be used for padding and conversion into a tensor of type
+ *torch.FloatTensor*.
attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
@@ -84,45 +86,45 @@ SPEECH_ENCODER_DECODER_INPUTS_DOCSTRING = r"""
decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
Indices of decoder input sequence tokens in the vocabulary.
- Indices can be obtained using [`PreTrainedTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`PreTrainedTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
`past_key_values`).
- For training, `decoder_input_ids` are automatically created by the model by shifting the `labels`
- to the right, replacing -100 by the `pad_token_id` and prepending them with the
- `decoder_start_token_id`.
+ For training, `decoder_input_ids` are automatically created by the model by shifting the `labels` to the
+ right, replacing -100 by the `pad_token_id` and prepending them with the `decoder_start_token_id`.
decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
- Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will
- also be used by default.
+ Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
+ be used by default.
encoder_outputs (`tuple(torch.FloatTensor)`, *optional*):
- This tuple must consist of (`last_hidden_state`, *optional*: `hidden_states`, *optional*:
- `attentions`) `last_hidden_state` (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`) is a tensor of hidden-states at the output of the last layer of the
- encoder. Used in the cross-attention of the decoder.
+ This tuple must consist of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`)
+ `last_hidden_state` (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`) is a tensor
+ of hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the
+ decoder.
past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ `decoder_input_ids` of shape `(batch_size, sequence_length)`.
inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*):
Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded
- representation. This is useful if you want more control over how to convert `decoder_input_ids`
- indices into associated vectors than the model's internal embedding lookup matrix.
+ representation. This is useful if you want more control over how to convert `decoder_input_ids` indices
+ into associated vectors than the model's internal embedding lookup matrix.
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss for the decoder. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
+ Labels for computing the masked language modeling loss for the decoder. Indices should be in `[-100, 0,
+ ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
(masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
use_cache (`bool`, *optional*):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`).
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`).
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -132,19 +134,16 @@ SPEECH_ENCODER_DECODER_INPUTS_DOCSTRING = r"""
input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
Float values of input raw speech waveform. Values can be obtained by loading a *.flac* or *.wav* audio file
into an array of type *List[float]* or a *numpy.ndarray*, *e.g.* via the soundfile library (*pip install
- soundfile*). To prepare the array into *input_values*, the [`Wav2Vec2Processor`] should
- be used for padding and conversion into a tensor of type *torch.FloatTensor*. See
- [`Wav2Vec2Processor.__call__`] for details.
+ soundfile*). To prepare the array into *input_values*, the [`Wav2Vec2Processor`] should be used for padding
+ and conversion into a tensor of type *torch.FloatTensor*. See [`Wav2Vec2Processor.__call__`] for details.
input_features (`torch.LongTensor` of shape `(batch_size, sequence_length, feature_size)`, *optional*):
Float values of fbank features extracted from the raw speech waveform. Raw speech waveform can be obtained
- by loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a
- `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the array
- into `input_features`, the [`Speech2TextTokenizer`] should be used for extracting
- the fbank features, padding and conversion into a tensor of type `torch.FloatTensor`. See
- [`~Speech2TextTokenizer.__call__`]
+ by loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a `numpy.ndarray`, *e.g.*
+ via the soundfile library (`pip install soundfile`). To prepare the array into `input_features`, the
+ [`Speech2TextTokenizer`] should be used for extracting the fbank features, padding and conversion into a
+ tensor of type `torch.FloatTensor`. See [`~Speech2TextTokenizer.__call__`]
return_dict (`bool`, *optional*):
- If set to `True`, the model will return a [`~file_utils.Seq2SeqLMOutput`] instead of a
- plain tuple.
+ If set to `True`, the model will return a [`~file_utils.Seq2SeqLMOutput`] instead of a plain tuple.
kwargs: (*optional*) Remaining dictionary of keyword arguments. Keyword arguments come in two flavors:
- Without a prefix which will be input as `**encoder_kwargs` for the encoder forward function.
@@ -174,9 +173,9 @@ def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int, decoder_start
@add_start_docstrings(SPEECH_ENCODER_DECODER_START_DOCSTRING)
class SpeechEncoderDecoderModel(PreTrainedModel):
r"""
- [`SpeechEncoderDecoderModel`] is a generic model class that will be instantiated as a
- transformer architecture with one of the base model classes of the library as encoder and another one as decoder
- when created with the :meth*~transformers.AutoModel.from_pretrained* class method for the encoder and
+ [`SpeechEncoderDecoderModel`] is a generic model class that will be instantiated as a transformer architecture with
+ one of the base model classes of the library as encoder and another one as decoder when created with the
+ :meth*~transformers.AutoModel.from_pretrained* class method for the encoder and
:meth*~transformers.AutoModelForCausalLM.from_pretrained* class method for the decoder.
"""
config_class = SpeechEncoderDecoderConfig
@@ -284,35 +283,35 @@ class SpeechEncoderDecoderModel(PreTrainedModel):
checkpoints.
- The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated). To
- train the model, you need to first set it back in training mode with `model.train()`.
+ The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated). To train
+ the model, you need to first set it back in training mode with `model.train()`.
Params:
encoder_pretrained_model_name_or_path (:obj: *str*, *optional*):
Information necessary to initiate the encoder. Can be either:
- A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co.
- Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under
- a user or organization name, like `dbmdz/bert-base-german-cased`.
+ Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under a
+ user or organization name, like `dbmdz/bert-base-german-cased`.
- A path to a *directory* containing model weights saved using
[`~PreTrainedModel.save_pretrained`], e.g., `./my_model_directory/`.
- A path or url to a *tensorflow index checkpoint file* (e.g, `./tf_model/model.ckpt.index`). In
- this case, `from_tf` should be set to `True` and a configuration object should be provided
- as `config` argument. This loading path is slower than converting the TensorFlow checkpoint in
- a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards.
+ this case, `from_tf` should be set to `True` and a configuration object should be provided as
+ `config` argument. This loading path is slower than converting the TensorFlow checkpoint in a
+ PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards.
decoder_pretrained_model_name_or_path (:obj: *str*, *optional*, defaults to *None*):
Information necessary to initiate the decoder. Can be either:
- A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co.
- Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under
- a user or organization name, like `dbmdz/bert-base-german-cased`.
+ Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under a
+ user or organization name, like `dbmdz/bert-base-german-cased`.
- A path to a *directory* containing model weights saved using
[`~PreTrainedModel.save_pretrained`], e.g., `./my_model_directory/`.
- A path or url to a *tensorflow index checkpoint file* (e.g, `./tf_model/model.ckpt.index`). In
- this case, `from_tf` should be set to `True` and a configuration object should be provided
- as `config` argument. This loading path is slower than converting the TensorFlow checkpoint in
- a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards.
+ this case, `from_tf` should be set to `True` and a configuration object should be provided as
+ `config` argument. This loading path is slower than converting the TensorFlow checkpoint in a
+ PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards.
model_args (remaining positional arguments, *optional*):
All remaning positional arguments will be passed to the underlying model's `__init__` method.
diff --git a/src/transformers/models/speech_to_text/configuration_speech_to_text.py b/src/transformers/models/speech_to_text/configuration_speech_to_text.py
index 4a6b165b4d..f08bbf51e1 100644
--- a/src/transformers/models/speech_to_text/configuration_speech_to_text.py
+++ b/src/transformers/models/speech_to_text/configuration_speech_to_text.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" Speech2Text model configuration """
+""" Speech2Text model configuration"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging
@@ -28,13 +28,13 @@ SPEECH_TO_TEXT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class Speech2TextConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`Speech2TextModel`]. It is used
- to instantiate an Speech2Text model according to the specified arguments, defining the model architecture.
- Instantiating a configuration with the defaults will yield a similar configuration to that of the Speech2Text
+ This is the configuration class to store the configuration of a [`Speech2TextModel`]. It is used to instantiate an
+ Speech2Text model according to the specified arguments, defining the model architecture. Instantiating a
+ configuration with the defaults will yield a similar configuration to that of the Speech2Text
[facebook/s2t-small-librispeech-asr](https://huggingface.co/facebook/s2t-small-librispeech-asr) architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
@@ -56,8 +56,8 @@ class Speech2TextConfig(PretrainedConfig):
encoder_ffn_dim (`int`, *optional*, defaults to 4096):
Dimensionality of the "intermediate" (often named feed-forward) layer in decoder.
activation_function (`str` or `function`, *optional*, defaults to `"gelu"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"silu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"silu"` and `"gelu_new"` are supported.
dropout (`float`, *optional*, defaults to 0.1):
The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
attention_dropout (`float`, *optional*, defaults to 0.0):
@@ -69,11 +69,11 @@ class Speech2TextConfig(PretrainedConfig):
init_std (`float`, *optional*, defaults to 0.02):
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
encoder_layerdrop: (`float`, *optional*, defaults to 0.0):
- The LayerDrop probability for the encoder. See the [LayerDrop paper](see
- https://arxiv.org/abs/1909.11556) for more details.
+ The LayerDrop probability for the encoder. See the [LayerDrop paper](see https://arxiv.org/abs/1909.11556)
+ for more details.
decoder_layerdrop: (`float`, *optional*, defaults to 0.0):
- The LayerDrop probability for the decoder. See the [LayerDrop paper](see
- https://arxiv.org/abs/1909.11556) for more details.
+ The LayerDrop probability for the decoder. See the [LayerDrop paper](see https://arxiv.org/abs/1909.11556)
+ for more details.
use_cache (`bool`, *optional*, defaults to `True`):
Whether or not the model should return the last key/values attentions (not used by all models).
max_source_positions (`int`, *optional*, defaults to 6000):
diff --git a/src/transformers/models/speech_to_text/feature_extraction_speech_to_text.py b/src/transformers/models/speech_to_text/feature_extraction_speech_to_text.py
index 8b924cde7b..8b621af685 100644
--- a/src/transformers/models/speech_to_text/feature_extraction_speech_to_text.py
+++ b/src/transformers/models/speech_to_text/feature_extraction_speech_to_text.py
@@ -35,8 +35,8 @@ class Speech2TextFeatureExtractor(SequenceFeatureExtractor):
r"""
Constructs a Speech2Text feature extractor.
- This feature extractor inherits from [`Speech2TextFeatureExtractor`] which contains most of the
- main methods. Users should refer to this superclass for more information regarding those methods.
+ This feature extractor inherits from [`Speech2TextFeatureExtractor`] which contains most of the main methods. Users
+ should refer to this superclass for more information regarding those methods.
This class extracts mel-filter bank features from raw speech using TorchAudio and applies utterance-level cepstral
mean and variance normalization to the extracted features.
@@ -147,12 +147,12 @@ class Speech2TextFeatureExtractor(SequenceFeatureExtractor):
Select a strategy to pad the returned sequences (according to the model's padding side and padding
index) among:
- - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a
- single sequence if provided).
- - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the
- maximum acceptable input length for the model if that argument is not provided.
- - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of
- different lengths).
+ - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
+ sequence if provided).
+ - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
+ acceptable input length for the model if that argument is not provided.
+ - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different
+ lengths).
max_length (`int`, *optional*):
Maximum length of the returned list and optionally padding length (see above).
truncation (`bool`):
@@ -170,8 +170,8 @@ class Speech2TextFeatureExtractor(SequenceFeatureExtractor):
- For Speech2TextTransoformer models, `attention_mask` should alwys be passed for batched
- inference, to avoid subtle bugs.
+ For Speech2TextTransoformer models, `attention_mask` should alwys be passed for batched inference, to
+ avoid subtle bugs.
diff --git a/src/transformers/models/speech_to_text/modeling_speech_to_text.py b/src/transformers/models/speech_to_text/modeling_speech_to_text.py
index 8fc2e83ed3..5264017423 100755
--- a/src/transformers/models/speech_to_text/modeling_speech_to_text.py
+++ b/src/transformers/models/speech_to_text/modeling_speech_to_text.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" PyTorch Speech2Text model. """
+""" PyTorch Speech2Text model."""
import math
@@ -464,7 +464,8 @@ class Speech2TextDecoderLayer(nn.Module):
hidden_states (`torch.FloatTensor`): input to the layer of shape `(seq_len, batch, embed_dim)`
attention_mask (`torch.FloatTensor`): attention mask of size
`(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
- encoder_hidden_states (`torch.FloatTensor`): cross attention input to the layer of shape `(seq_len, batch, embed_dim)`
+ encoder_hidden_states (`torch.FloatTensor`):
+ cross attention input to the layer of shape `(seq_len, batch, embed_dim)`
encoder_attention_mask (`torch.FloatTensor`): encoder attention mask of size
`(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
layer_head_mask (`torch.FloatTensor`): mask for attention heads in a given layer of size
@@ -586,13 +587,13 @@ class Speech2TextPreTrainedModel(PreTrainedModel):
SPEECH_TO_TEXT_START_DOCSTRING = r"""
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
Parameters:
config ([`Speech2TextConfig`]):
@@ -605,13 +606,13 @@ SPEECH_TO_TEXT_INPUTS_DOCSTRING = r"""
Args:
input_features (`torch.LongTensor` of shape `(batch_size, sequence_length, feature_size)`):
Float values of fbank features extracted from the raw speech waveform. Raw speech waveform can be obtained
- by loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a
- `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the array
- into `input_features`, the [`Speech2TextTokenizer`] should be used for extracting
- the fbank features, padding and conversion into a tensor of type `torch.FloatTensor`. See
- [`~Speech2TextTokenizer.__call__`]
+ by loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a `numpy.ndarray`, *e.g.*
+ via the soundfile library (`pip install soundfile`). To prepare the array into `input_features`, the
+ [`Speech2TextTokenizer`] should be used for extracting the fbank features, padding and conversion into a
+ tensor of type `torch.FloatTensor`. See [`~Speech2TextTokenizer.__call__`]
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
- Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0, 1]`:
+ Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0,
+ 1]`:
- 1 for tokens that are **not masked**,
- 0 for tokens that are **masked**.
@@ -620,9 +621,8 @@ SPEECH_TO_TEXT_INPUTS_DOCSTRING = r"""
decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
Indices of decoder input sequence tokens in the vocabulary.
- Indices can be obtained using [`SpeechToTextTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`SpeechToTextTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are decoder input IDs?](../glossary#decoder-input-ids)
@@ -630,12 +630,12 @@ SPEECH_TO_TEXT_INPUTS_DOCSTRING = r"""
`past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
`past_key_values`).
decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
- Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will
- also be used by default.
+ Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
+ be used by default.
- If you want to change padding behavior, you should read
- [`modeling_speech_to_text._prepare_decoder_inputs`] and modify to your needs. See diagram 1 in [the
- paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy.
+ If you want to change padding behavior, you should read [`modeling_speech_to_text._prepare_decoder_inputs`]
+ and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more
+ information on the default strategy.
head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*):
Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`:
@@ -655,27 +655,31 @@ SPEECH_TO_TEXT_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*):
- Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*:
- `attentions`) `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`,
- *optional*) is a sequence of hidden-states at the output of the last layer of the encoder. Used in the
- cross-attention of the decoder.
+ Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`)
+ `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of
+ hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder.
past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors
- of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
- shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
+ Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
+ `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape
+ `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all ``decoder_input_ids``` of shape `(batch_size, sequence_length)`. decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be input (see `past_key_values`). This is useful if you want more control over how to convert `decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ ``decoder_input_ids``` of shape `(batch_size, sequence_length)`. decoder_inputs_embeds (`torch.FloatTensor`
+ of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*): Optionally, instead of passing
+ `decoder_input_ids` you can choose to directly pass an embedded representation. If `past_key_values` is
+ used, optionally only the last `decoder_inputs_embeds` have to be input (see `past_key_values`). This is
+ useful if you want more control over how to convert `decoder_input_ids` indices into associated vectors
+ than the model's internal embedding lookup matrix.
- If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds`
- takes the value of `inputs_embeds`.
+ If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds` takes the value
+ of `inputs_embeds`.
use_cache (`bool`, *optional*):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`).
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`).
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -736,10 +740,10 @@ class Speech2TextEncoder(Speech2TextPreTrainedModel):
input_features (`torch.LongTensor` of shape `(batch_size, sequence_length, feature_size)`):
Float values of fbank features extracted from the raw speech waveform. Raw speech waveform can be
obtained by loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a
- `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the
- array into `input_features`, the [`Speech2TextTokenizer`] should be used for
- extracting the fbank features, padding and conversion into a tensor of type `torch.FloatTensor`.
- See [`~Speech2TextTokenizer.__call__`]
+ `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the array into
+ `input_features`, the [`Speech2TextTokenizer`] should be used for extracting the fbank features,
+ padding and conversion into a tensor of type `torch.FloatTensor`. See
+ [`~Speech2TextTokenizer.__call__`]
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
Mask to avoid performing convolution and attention on padding token indices. Mask values selected in
`[0, 1]`:
@@ -921,9 +925,8 @@ class Speech2TextDecoder(Speech2TextPreTrainedModel):
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
provide it.
- Indices can be obtained using [`Speech2TextTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`]
- for details.
+ Indices can be obtained using [`Speech2TextTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -958,19 +961,20 @@ class Speech2TextDecoder(Speech2TextPreTrainedModel):
- 0 indicates the head is **masked**.
past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2
- tensors of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional
- tensors of shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
+ Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of
+ shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
+ shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
Contains pre-computed hidden-states (key and values in the self-attention blocks and in the
- cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential
- decoding.
+ cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
- If `past_key_values` are used, the user can optionally input only the last
- `decoder_input_ids` (those that don't have their past key value states given to this model) of
- shape `(batch_size, 1)` instead of all ``decoder_input_ids``` of shape `(batch_size,
- sequence_length)`. inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices
- into associated vectors than the model's internal embedding lookup matrix.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those
+ that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of
+ all ``decoder_input_ids``` of shape `(batch_size, sequence_length)`. inputs_embeds (`torch.FloatTensor`
+ of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing
+ `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more
+ control over how to convert `input_ids` indices into associated vectors than the model's internal
+ embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under
returned tensors for more detail.
@@ -1289,8 +1293,9 @@ class Speech2TextForConditionalGeneration(Speech2TextPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
+ Labels for computing the language modeling loss. Indices should either be in `[0, ..., config.vocab_size]`
+ or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is
+ only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
Returns:
diff --git a/src/transformers/models/speech_to_text/processing_speech_to_text.py b/src/transformers/models/speech_to_text/processing_speech_to_text.py
index 2db1546f89..5eb9791cc1 100644
--- a/src/transformers/models/speech_to_text/processing_speech_to_text.py
+++ b/src/transformers/models/speech_to_text/processing_speech_to_text.py
@@ -26,15 +26,13 @@ class Speech2TextProcessor:
Constructs a Speech2Text processor which wraps a Speech2Text feature extractor and a Speech2Text tokenizer into a
single processor.
- [`Speech2TextProcessor`] offers all the functionalities of
- [`Speech2TextFeatureExtractor`] and [`Speech2TextTokenizer`]. See the
- [`~Speech2TextProcessor.__call__`] and [`~Speech2TextProcessor.decode`] for more
+ [`Speech2TextProcessor`] offers all the functionalities of [`Speech2TextFeatureExtractor`] and
+ [`Speech2TextTokenizer`]. See the [`~Speech2TextProcessor.__call__`] and [`~Speech2TextProcessor.decode`] for more
information.
Args:
feature_extractor (`Speech2TextFeatureExtractor`):
- An instance of [`Speech2TextFeatureExtractor`]. The feature extractor is a required
- input.
+ An instance of [`Speech2TextFeatureExtractor`]. The feature extractor is a required input.
tokenizer (`Speech2TextTokenizer`):
An instance of [`Speech2TextTokenizer`]. The tokenizer is a required input.
"""
@@ -55,15 +53,14 @@ class Speech2TextProcessor:
def save_pretrained(self, save_directory):
"""
- Save a Speech2Text feature extractor object and Speech2Text tokenizer object to the directory
- `save_directory`, so that it can be re-loaded using the
- [`~Speech2TextProcessor.from_pretrained`] class method.
+ Save a Speech2Text feature extractor object and Speech2Text tokenizer object to the directory `save_directory`,
+ so that it can be re-loaded using the [`~Speech2TextProcessor.from_pretrained`] class method.
This class method is simply calling [`~PreTrainedFeatureExtractor.save_pretrained`] and
- [`~tokenization_utils_base.PreTrainedTokenizer.save_pretrained`]. Please refer to the
- docstrings of the methods above for more information.
+ [`~tokenization_utils_base.PreTrainedTokenizer.save_pretrained`]. Please refer to the docstrings of the methods
+ above for more information.
@@ -85,8 +82,8 @@ class Speech2TextProcessor:
This class method is simply calling Speech2TextFeatureExtractor's
[`~PreTrainedFeatureExtractor.from_pretrained`] and Speech2TextTokenizer's
- [`~tokenization_utils_base.PreTrainedTokenizer.from_pretrained`]. Please refer to the
- docstrings of the methods above for more information.
+ [`~tokenization_utils_base.PreTrainedTokenizer.from_pretrained`]. Please refer to the docstrings of the methods
+ above for more information.
@@ -98,8 +95,7 @@ class Speech2TextProcessor:
huggingface.co. Valid model ids can be located at the root-level, like `bert-base-uncased`, or
namespaced under a user or organization name, like `dbmdz/bert-base-german-cased`.
- a path to a *directory* containing a feature extractor file saved using the
- [`~PreTrainedFeatureExtractor.save_pretrained`] method, e.g.,
- `./my_model_directory/`.
+ [`~PreTrainedFeatureExtractor.save_pretrained`] method, e.g., `./my_model_directory/`.
- a path or url to a saved feature extractor JSON *file*, e.g.,
`./my_model_directory/preprocessor_config.json`.
**kwargs
@@ -115,25 +111,23 @@ class Speech2TextProcessor:
"""
When used in normal mode, this method forwards all its arguments to Speech2TextFeatureExtractor's
[`~Speech2TextFeatureExtractor.__call__`] and returns its output. If used in the context
- [`~Speech2TextProcessor.as_target_processor`] this method forwards all its arguments to
- Speech2TextTokenizer's [`~Speech2TextTokenizer.__call__`]. Please refer to the doctsring of
- the above two methods for more information.
+ [`~Speech2TextProcessor.as_target_processor`] this method forwards all its arguments to Speech2TextTokenizer's
+ [`~Speech2TextTokenizer.__call__`]. Please refer to the doctsring of the above two methods for more
+ information.
"""
return self.current_processor(*args, **kwargs)
def batch_decode(self, *args, **kwargs):
"""
- This method forwards all its arguments to Speech2TextTokenizer's
- [`~PreTrainedTokenizer.batch_decode`]. Please refer to the docstring of this method for more
- information.
+ This method forwards all its arguments to Speech2TextTokenizer's [`~PreTrainedTokenizer.batch_decode`]. Please
+ refer to the docstring of this method for more information.
"""
return self.tokenizer.batch_decode(*args, **kwargs)
def decode(self, *args, **kwargs):
"""
- This method forwards all its arguments to Speech2TextTokenizer's
- [`~PreTrainedTokenizer.decode`]. Please refer to the docstring of this method for more
- information.
+ This method forwards all its arguments to Speech2TextTokenizer's [`~PreTrainedTokenizer.decode`]. Please refer
+ to the docstring of this method for more information.
"""
return self.tokenizer.decode(*args, **kwargs)
diff --git a/src/transformers/models/speech_to_text/tokenization_speech_to_text.py b/src/transformers/models/speech_to_text/tokenization_speech_to_text.py
index aeef9d224d..cf40fa713d 100644
--- a/src/transformers/models/speech_to_text/tokenization_speech_to_text.py
+++ b/src/transformers/models/speech_to_text/tokenization_speech_to_text.py
@@ -56,8 +56,8 @@ class Speech2TextTokenizer(PreTrainedTokenizer):
"""
Construct an Speech2Text tokenizer.
- This tokenizer inherits from [`PreTrainedTokenizer`] which contains some of the main methods.
- Users should refer to the superclass for more information regarding such methods.
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains some of the main methods. Users should refer to
+ the superclass for more information regarding such methods.
Args:
vocab_file (`str`):
@@ -80,7 +80,9 @@ class Speech2TextTokenizer(PreTrainedTokenizer):
tgt_lang (`str`, *optional*):
A string representing the target language.
sp_model_kwargs (`dict`, *optional*):
- Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things, to set:
+ Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for
+ SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things,
+ to set:
- `enable_sampling`: Enable subword regularization.
- `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout.
diff --git a/src/transformers/models/speech_to_text_2/configuration_speech_to_text_2.py b/src/transformers/models/speech_to_text_2/configuration_speech_to_text_2.py
index 15579c459d..13cd724867 100644
--- a/src/transformers/models/speech_to_text_2/configuration_speech_to_text_2.py
+++ b/src/transformers/models/speech_to_text_2/configuration_speech_to_text_2.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" Speech2Text model configuration """
+""" Speech2Text model configuration"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging
@@ -28,13 +28,13 @@ SPEECH_TO_TEXT_2_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class Speech2Text2Config(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`Speech2Text2ForCausalLM`]. It
- is used to instantiate an Speech2Text2 model according to the specified arguments, defining the model architecture.
+ This is the configuration class to store the configuration of a [`Speech2Text2ForCausalLM`]. It is used to
+ instantiate an Speech2Text2 model according to the specified arguments, defining the model architecture.
Instantiating a configuration with the defaults will yield a similar configuration to that of the Speech2Text2
[facebook/s2t-small-librispeech-asr](https://huggingface.co/facebook/s2t-small-librispeech-asr) architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
@@ -50,8 +50,8 @@ class Speech2Text2Config(PretrainedConfig):
decoder_ffn_dim (`int`, *optional*, defaults to 4096):
Dimensionality of the "intermediate" (often named feed-forward) layer in decoder.
activation_function (`str` or `function`, *optional*, defaults to `"gelu"`):
- The non-linear activation function (function or string) in the pooler. If string, `"gelu"`,
- `"relu"`, `"silu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the pooler. If string, `"gelu"`, `"relu"`,
+ `"silu"` and `"gelu_new"` are supported.
dropout (`float`, *optional*, defaults to 0.1):
The dropout probability for all fully connected layers in the embeddings, and pooler.
attention_dropout (`float`, *optional*, defaults to 0.0):
@@ -62,8 +62,8 @@ class Speech2Text2Config(PretrainedConfig):
The dropout ratio for classifier.
init_std (`float`, *optional*, defaults to 0.02):
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
- https://arxiv.org/abs/1909.11556>`__ for more details. decoder_layerdrop: (`float`, *optional*, defaults to 0.0):
- The LayerDrop probability for the decoder. See the [LayerDrop paper](see
+ https://arxiv.org/abs/1909.11556>`__ for more details. decoder_layerdrop: (`float`, *optional*, defaults to
+ 0.0): The LayerDrop probability for the decoder. See the [LayerDrop paper](see
https://arxiv.org/abs/1909.11556) for more details.
use_cache (`bool`, *optional*, defaults to `True`):
Whether or not the model should return the last key/values attentions (not used by all models).
diff --git a/src/transformers/models/speech_to_text_2/modeling_speech_to_text_2.py b/src/transformers/models/speech_to_text_2/modeling_speech_to_text_2.py
index bdcf90b65e..9d09253203 100755
--- a/src/transformers/models/speech_to_text_2/modeling_speech_to_text_2.py
+++ b/src/transformers/models/speech_to_text_2/modeling_speech_to_text_2.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" PyTorch Speech2Text2 model. """
+""" PyTorch Speech2Text2 model."""
import copy
@@ -339,7 +339,8 @@ class Speech2Text2DecoderLayer(nn.Module):
hidden_states (`torch.FloatTensor`): input to the layer of shape `(seq_len, batch, embed_dim)`
attention_mask (`torch.FloatTensor`): attention mask of size
`(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
- encoder_hidden_states (`torch.FloatTensor`): cross attention input to the layer of shape `(seq_len, batch, embed_dim)`
+ encoder_hidden_states (`torch.FloatTensor`):
+ cross attention input to the layer of shape `(seq_len, batch, embed_dim)`
encoder_attention_mask (`torch.FloatTensor`): encoder attention mask of size
`(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
layer_head_mask (`torch.FloatTensor`): mask for attention heads in a given layer of size
@@ -433,13 +434,13 @@ class Speech2Text2PreTrainedModel(PreTrainedModel):
SPEECH_TO_TEXT_2_START_DOCSTRING = r"""
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
Parameters:
config ([`Speech2Text2Config`]):
@@ -525,9 +526,8 @@ class Speech2Text2Decoder(Speech2Text2PreTrainedModel):
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
provide it.
- Indices can be obtained using [`Speech2Text2Tokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`]
- for details.
+ Indices can be obtained using [`Speech2Text2Tokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -562,19 +562,20 @@ class Speech2Text2Decoder(Speech2Text2PreTrainedModel):
- 0 indicates the head is **masked**.
past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2
- tensors of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional
- tensors of shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
+ Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of
+ shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
+ shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
Contains pre-computed hidden-states (key and values in the self-attention blocks and in the
- cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential
- decoding.
+ cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
- If `past_key_values` are used, the user can optionally input only the last
- `decoder_input_ids` (those that don't have their past key value states given to this model) of
- shape `(batch_size, 1)` instead of all ``decoder_input_ids``` of shape `(batch_size,
- sequence_length)`. inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices
- into associated vectors than the model's internal embedding lookup matrix.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those
+ that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of
+ all ``decoder_input_ids``` of shape `(batch_size, sequence_length)`. inputs_embeds (`torch.FloatTensor`
+ of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing
+ `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more
+ control over how to convert `input_ids` indices into associated vectors than the model's internal
+ embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under
returned tensors for more detail.
@@ -793,9 +794,8 @@ class Speech2Text2ForCausalLM(Speech2Text2PreTrainedModel):
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
provide it.
- Indices can be obtained using [`Speech2Text2Tokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`]
- for details.
+ Indices can be obtained using [`Speech2Text2Tokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -824,25 +824,24 @@ class Speech2Text2ForCausalLM(Speech2Text2PreTrainedModel):
- 0 indicates the head is **masked**.
past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2
- tensors of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional
- tensors of shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. The two
- additional tensors are only required when the model is used as a decoder in a Sequence to Sequence
- model.
+ Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of
+ shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
+ shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. The two additional
+ tensors are only required when the model is used as a decoder in a Sequence to Sequence model.
Contains pre-computed hidden-states (key and values in the self-attention blocks and in the
- cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential
- decoding.
+ cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those
+ that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of
+ all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are
- ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
+ Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
+ config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
+ (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
use_cache (`bool`, *optional*):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`).
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
+ (see `past_key_values`).
- 1 for tokens that are **not masked**,
- 0 for tokens that are **masked**.
diff --git a/src/transformers/models/speech_to_text_2/processing_speech_to_text_2.py b/src/transformers/models/speech_to_text_2/processing_speech_to_text_2.py
index 1d6c260793..5567ef982b 100644
--- a/src/transformers/models/speech_to_text_2/processing_speech_to_text_2.py
+++ b/src/transformers/models/speech_to_text_2/processing_speech_to_text_2.py
@@ -27,10 +27,8 @@ class Speech2Text2Processor:
Constructs a Speech2Text2 processor which wraps a Speech2Text2 feature extractor and a Speech2Text2 tokenizer into
a single processor.
- [`Speech2Text2Processor`] offers all the functionalities of
- [`AutoFeatureExtractor`] and [`Speech2Text2Tokenizer`]. See the
- [`~Speech2Text2Processor.__call__`] and [`~Speech2Text2Processor.decode`] for
- more information.
+ [`Speech2Text2Processor`] offers all the functionalities of [`AutoFeatureExtractor`] and [`Speech2Text2Tokenizer`].
+ See the [`~Speech2Text2Processor.__call__`] and [`~Speech2Text2Processor.decode`] for more information.
Args:
feature_extractor (`AutoFeatureExtractor`):
@@ -56,14 +54,14 @@ class Speech2Text2Processor:
def save_pretrained(self, save_directory):
"""
Save a Speech2Text2 feature extractor object and Speech2Text2 tokenizer object to the directory
- `save_directory`, so that it can be re-loaded using the
- [`~Speech2Text2Processor.from_pretrained`] class method.
+ `save_directory`, so that it can be re-loaded using the [`~Speech2Text2Processor.from_pretrained`] class
+ method.
This class method is simply calling [`~PreTrainedFeatureExtractor.save_pretrained`] and
- [`~tokenization_utils_base.PreTrainedTokenizer.save_pretrained`]. Please refer to the
- docstrings of the methods above for more information.
+ [`~tokenization_utils_base.PreTrainedTokenizer.save_pretrained`]. Please refer to the docstrings of the methods
+ above for more information.
@@ -83,9 +81,8 @@ class Speech2Text2Processor:
- This class method is simply calling AutoFeatureExtractor's
- [`~PreTrainedFeatureExtractor.from_pretrained`] and Speech2Text2Tokenizer's
- [`~tokenization_utils_base.PreTrainedTokenizer.from_pretrained`]. Please refer to the
+ This class method is simply calling AutoFeatureExtractor's [`~PreTrainedFeatureExtractor.from_pretrained`] and
+ Speech2Text2Tokenizer's [`~tokenization_utils_base.PreTrainedTokenizer.from_pretrained`]. Please refer to the
docstrings of the methods above for more information.
@@ -98,8 +95,7 @@ class Speech2Text2Processor:
huggingface.co. Valid model ids can be located at the root-level, like `bert-base-uncased`, or
namespaced under a user or organization name, like `dbmdz/bert-base-german-cased`.
- a path to a *directory* containing a feature extractor file saved using the
- [`~PreTrainedFeatureExtractor.save_pretrained`] method, e.g.,
- `./my_model_directory/`.
+ [`~PreTrainedFeatureExtractor.save_pretrained`] method, e.g., `./my_model_directory/`.
- a path or url to a saved feature extractor JSON *file*, e.g.,
`./my_model_directory/preprocessor_config.json`.
**kwargs
@@ -116,24 +112,22 @@ class Speech2Text2Processor:
When used in normal mode, this method forwards all its arguments to AutoFeatureExtractor's
[`~AutoFeatureExtractor.__call__`] and returns its output. If used in the context
[`~Speech2Text2Processor.as_target_processor`] this method forwards all its arguments to
- Speech2Text2Tokenizer's [`~Speech2Text2Tokenizer.__call__`]. Please refer to the doctsring of
- the above two methods for more information.
+ Speech2Text2Tokenizer's [`~Speech2Text2Tokenizer.__call__`]. Please refer to the doctsring of the above two
+ methods for more information.
"""
return self.current_processor(*args, **kwargs)
def batch_decode(self, *args, **kwargs):
"""
- This method forwards all its arguments to Speech2Text2Tokenizer's
- [`~PreTrainedTokenizer.batch_decode`]. Please refer to the docstring of this method for more
- information.
+ This method forwards all its arguments to Speech2Text2Tokenizer's [`~PreTrainedTokenizer.batch_decode`]. Please
+ refer to the docstring of this method for more information.
"""
return self.tokenizer.batch_decode(*args, **kwargs)
def decode(self, *args, **kwargs):
"""
- This method forwards all its arguments to Speech2Text2Tokenizer's
- [`~PreTrainedTokenizer.decode`]. Please refer to the docstring of this method for more
- information.
+ This method forwards all its arguments to Speech2Text2Tokenizer's [`~PreTrainedTokenizer.decode`]. Please refer
+ to the docstring of this method for more information.
"""
return self.tokenizer.decode(*args, **kwargs)
diff --git a/src/transformers/models/speech_to_text_2/tokenization_speech_to_text_2.py b/src/transformers/models/speech_to_text_2/tokenization_speech_to_text_2.py
index fc27a53a76..51d5c31ec9 100644
--- a/src/transformers/models/speech_to_text_2/tokenization_speech_to_text_2.py
+++ b/src/transformers/models/speech_to_text_2/tokenization_speech_to_text_2.py
@@ -68,8 +68,8 @@ class Speech2Text2Tokenizer(PreTrainedTokenizer):
"""
Constructs a Speech2Text2Tokenizer.
- This tokenizer inherits from [`PreTrainedTokenizer`] which contains some of the main methods.
- Users should refer to the superclass for more information regarding such methods.
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains some of the main methods. Users should refer to
+ the superclass for more information regarding such methods.
Args:
vocab_file (`str`):
diff --git a/src/transformers/models/splinter/configuration_splinter.py b/src/transformers/models/splinter/configuration_splinter.py
index f7f5ca0a0e..60b2580bec 100644
--- a/src/transformers/models/splinter/configuration_splinter.py
+++ b/src/transformers/models/splinter/configuration_splinter.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" Splinter model configuration """
+""" Splinter model configuration"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging
@@ -31,12 +31,13 @@ SPLINTER_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class SplinterConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`SplinterModel`]. It is used to
- instantiate an Splinter model according to the specified arguments, defining the model architecture. Instantiating
- a configuration with the defaults will yield a similar configuration to that of the Splinter [tau/splinter-base](https://huggingface.co/tau/splinter-base) architecture.
+ This is the configuration class to store the configuration of a [`SplinterModel`]. It is used to instantiate an
+ Splinter model according to the specified arguments, defining the model architecture. Instantiating a configuration
+ with the defaults will yield a similar configuration to that of the Splinter
+ [tau/splinter-base](https://huggingface.co/tau/splinter-base) architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
@@ -52,8 +53,8 @@ class SplinterConfig(PretrainedConfig):
intermediate_size (`int`, *optional*, defaults to 3072):
Dimension of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"selu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"selu"` and `"gelu_new"` are supported.
hidden_dropout_prob (`float`, *optional*, defaults to 0.1):
The dropout probabilitiy for all fully connected layers in the embeddings, encoder, and pooler.
attention_probs_dropout_prob (`float`, *optional*, defaults to 0.1):
diff --git a/src/transformers/models/splinter/modeling_splinter.py b/src/transformers/models/splinter/modeling_splinter.py
index 7abb71419d..b982a38b62 100755
--- a/src/transformers/models/splinter/modeling_splinter.py
+++ b/src/transformers/models/splinter/modeling_splinter.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" PyTorch Splinter model. """
+""" PyTorch Splinter model."""
import math
@@ -546,8 +546,7 @@ SPLINTER_START_DOCSTRING = r"""
Parameters:
config ([`SplinterConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
SPLINTER_INPUTS_DOCSTRING = r"""
@@ -555,9 +554,8 @@ SPLINTER_INPUTS_DOCSTRING = r"""
input_ids (`torch.LongTensor` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`SplinterTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`SplinterTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.FloatTensor` of shape `{0}`, *optional*):
@@ -568,14 +566,16 @@ SPLINTER_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`torch.LongTensor` of shape `{0}`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`torch.LongTensor` of shape `{0}`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
@@ -585,9 +585,9 @@ SPLINTER_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert *input_ids* indices into associated vectors
- than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -606,8 +606,8 @@ SPLINTER_INPUTS_DOCSTRING = r"""
class SplinterModel(SplinterPreTrainedModel):
"""
The model is an encoder (with only self-attention) following the architecture described in [Attention is all you
- need](https://arxiv.org/abs/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion
- Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin.
+ need](https://arxiv.org/abs/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones,
+ Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin.
"""
def __init__(self, config):
@@ -669,12 +669,12 @@ class SplinterModel(SplinterPreTrainedModel):
- 0 for tokens that are **masked**.
past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ `decoder_input_ids` of shape `(batch_size, sequence_length)`.
use_cache (`bool`, *optional*):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`).
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`).
"""
output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
output_hidden_states = (
@@ -861,17 +861,17 @@ class SplinterForQuestionAnswering(SplinterPreTrainedModel):
r"""
start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the start of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the end of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
question_positions (`torch.LongTensor` of shape `(batch_size, num_questions)`, *optional*):
- The positions of all question tokens. If given, start_logits and end_logits will be of shape
- `(batch_size, num_questions, sequence_length)`. If None, the first question token in each sequence in
- the batch will be the only one for which start_logits and end_logits are calculated and they will be of
- shape `(batch_size, sequence_length)`.
+ The positions of all question tokens. If given, start_logits and end_logits will be of shape `(batch_size,
+ num_questions, sequence_length)`. If None, the first question token in each sequence in the batch will be
+ the only one for which start_logits and end_logits are calculated and they will be of shape `(batch_size,
+ sequence_length)`.
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
diff --git a/src/transformers/models/splinter/tokenization_splinter.py b/src/transformers/models/splinter/tokenization_splinter.py
index f20b046820..9649da03f9 100644
--- a/src/transformers/models/splinter/tokenization_splinter.py
+++ b/src/transformers/models/splinter/tokenization_splinter.py
@@ -76,8 +76,8 @@ class SplinterTokenizer(PreTrainedTokenizer):
r"""
Construct a Splinter tokenizer. Based on WordPiece.
- This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods.
- Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
+ this superclass for more information regarding those methods.
Args:
vocab_file (`str`):
@@ -109,7 +109,8 @@ class SplinterTokenizer(PreTrainedTokenizer):
tokenize_chinese_chars (`bool`, *optional*, defaults to `True`):
Whether or not to tokenize Chinese characters.
- This should likely be deactivated for Japanese (see this [issue](https://github.com/huggingface/transformers/issues/328)).
+ This should likely be deactivated for Japanese (see this
+ [issue](https://github.com/huggingface/transformers/issues/328)).
strip_accents: (`bool`, *optional*):
Whether or not to strip all accents. If this option is not specified, then it will be determined by the
value for `lowercase` (as in the original BERT).
@@ -278,7 +279,8 @@ class SplinterTokenizer(PreTrainedTokenizer):
self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
) -> List[int]:
"""
- Create the token type IDs corresponding to the sequences passed. [What are token type IDs?](../glossary#token-type-ids)
+ Create the token type IDs corresponding to the sequences passed. [What are token type
+ IDs?](../glossary#token-type-ids)
Should be overridden in a subclass if the model has a special way of building those.
@@ -336,7 +338,8 @@ class BasicTokenizer(object):
tokenize_chinese_chars (`bool`, *optional*, defaults to `True`):
Whether or not to tokenize Chinese characters.
- This should likely be deactivated for Japanese (see this [issue](https://github.com/huggingface/transformers/issues/328)).
+ This should likely be deactivated for Japanese (see this
+ [issue](https://github.com/huggingface/transformers/issues/328)).
strip_accents: (`bool`, *optional*):
Whether or not to strip all accents. If this option is not specified, then it will be determined by the
value for `lowercase` (as in the original BERT).
diff --git a/src/transformers/models/splinter/tokenization_splinter_fast.py b/src/transformers/models/splinter/tokenization_splinter_fast.py
index dd50534234..103ead72ae 100644
--- a/src/transformers/models/splinter/tokenization_splinter_fast.py
+++ b/src/transformers/models/splinter/tokenization_splinter_fast.py
@@ -56,8 +56,8 @@ class SplinterTokenizerFast(PreTrainedTokenizerFast):
r"""
Construct a "fast" Splinter tokenizer (backed by HuggingFace's *tokenizers* library). Based on WordPiece.
- This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main
- methods. Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should
+ refer to this superclass for more information regarding those methods.
Args:
vocab_file (`str`):
@@ -186,7 +186,8 @@ class SplinterTokenizerFast(PreTrainedTokenizerFast):
self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
) -> List[int]:
"""
- Create the token type IDs corresponding to the sequences passed. [What are token type IDs?](../glossary#token-type-ids)
+ Create the token type IDs corresponding to the sequences passed. [What are token type
+ IDs?](../glossary#token-type-ids)
Should be overridden in a subclass if the model has a special way of building those.
diff --git a/src/transformers/models/squeezebert/configuration_squeezebert.py b/src/transformers/models/squeezebert/configuration_squeezebert.py
index 7a8930bfc1..d53ddaa916 100644
--- a/src/transformers/models/squeezebert/configuration_squeezebert.py
+++ b/src/transformers/models/squeezebert/configuration_squeezebert.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" SqueezeBERT model configuration """
+""" SqueezeBERT model configuration"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging
@@ -29,11 +29,11 @@ SQUEEZEBERT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class SqueezeBertConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`SqueezeBertModel`]. It is used
- to instantiate a SqueezeBERT model according to the specified arguments, defining the model architecture.
+ This is the configuration class to store the configuration of a [`SqueezeBertModel`]. It is used to instantiate a
+ SqueezeBERT model according to the specified arguments, defining the model architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
@@ -49,8 +49,8 @@ class SqueezeBertConfig(PretrainedConfig):
intermediate_size (`int`, *optional*, defaults to 3072):
Dimensionality of the "intermediate" (often named feed-forward) layer in the Transformer encoder.
hidden_act (`str` or `Callable`, *optional*, defaults to `"gelu"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"silu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"silu"` and `"gelu_new"` are supported.
hidden_dropout_prob (`float`, *optional*, defaults to 0.1):
The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
attention_probs_dropout_prob (`float`, *optional*, defaults to 0.1):
@@ -59,8 +59,7 @@ class SqueezeBertConfig(PretrainedConfig):
The maximum sequence length that this model might ever be used with. Typically set this to something large
just in case (e.g., 512 or 1024 or 2048).
type_vocab_size (`int`, *optional*, defaults to 2):
- The vocabulary size of the `token_type_ids` passed when calling [`BertModel`] or
- [`TFBertModel`].
+ The vocabulary size of the `token_type_ids` passed when calling [`BertModel`] or [`TFBertModel`].
initializer_range (`float`, *optional*, defaults to 0.02):
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
layer_norm_eps (`float`, *optional*, defaults to 1e-12):
diff --git a/src/transformers/models/squeezebert/modeling_squeezebert.py b/src/transformers/models/squeezebert/modeling_squeezebert.py
index dfcd42ea20..75ec95a2b8 100644
--- a/src/transformers/models/squeezebert/modeling_squeezebert.py
+++ b/src/transformers/models/squeezebert/modeling_squeezebert.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" PyTorch SqueezeBert model. """
+""" PyTorch SqueezeBert model."""
import math
@@ -453,13 +453,13 @@ SQUEEZEBERT_START_DOCSTRING = r"""
networks?](https://arxiv.org/abs/2006.11316) by Forrest N. Iandola, Albert E. Shaw, Ravi Krishna, and Kurt W.
Keutzer
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
For best results finetuning SqueezeBERT on text classification tasks, it is recommended to use the
*squeezebert/squeezebert-mnli-headless* checkpoint as a starting point.
@@ -467,8 +467,7 @@ SQUEEZEBERT_START_DOCSTRING = r"""
Parameters:
config ([`SqueezeBertConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
Hierarchy:
@@ -498,9 +497,8 @@ SQUEEZEBERT_INPUTS_DOCSTRING = r"""
input_ids (`torch.LongTensor` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`SqueezeBertTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`SqueezeBertTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
@@ -511,14 +509,16 @@ SQUEEZEBERT_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
@@ -528,9 +528,9 @@ SQUEEZEBERT_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -645,7 +645,7 @@ class SqueezeBertModel(SqueezeBertPreTrainedModel):
)
-@add_start_docstrings("""SqueezeBERT Model with a `language modeling` head on top. """, SQUEEZEBERT_START_DOCSTRING)
+@add_start_docstrings("""SqueezeBERT Model with a `language modeling` head on top.""", SQUEEZEBERT_START_DOCSTRING)
class SqueezeBertForMaskedLM(SqueezeBertPreTrainedModel):
_keys_to_ignore_on_load_missing = [r"predictions.decoder.bias"]
@@ -687,8 +687,9 @@ class SqueezeBertForMaskedLM(SqueezeBertPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -766,8 +767,9 @@ class SqueezeBertForSequenceClassification(SqueezeBertPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -865,7 +867,8 @@ class SqueezeBertForMultipleChoice(SqueezeBertPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where *num_choices* is the size of the second dimension of the input tensors. (see
+ Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
+ num_choices-1]` where *num_choices* is the size of the second dimension of the input tensors. (see
*input_ids* above)
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
diff --git a/src/transformers/models/squeezebert/tokenization_squeezebert.py b/src/transformers/models/squeezebert/tokenization_squeezebert.py
index 2b89f724c0..e41e576455 100644
--- a/src/transformers/models/squeezebert/tokenization_squeezebert.py
+++ b/src/transformers/models/squeezebert/tokenization_squeezebert.py
@@ -48,11 +48,10 @@ class SqueezeBertTokenizer(BertTokenizer):
r"""
Constructs a SqueezeBert tokenizer.
- [`SqueezeBertTokenizer`] is identical to [`BertTokenizer`] and runs end-to-end
- tokenization: punctuation splitting + wordpiece.
+ [`SqueezeBertTokenizer`] is identical to [`BertTokenizer`] and runs end-to-end tokenization: punctuation splitting
+ + wordpiece.
- Refer to superclass [`BertTokenizer`] for usage examples and documentation concerning
- parameters.
+ Refer to superclass [`BertTokenizer`] for usage examples and documentation concerning parameters.
"""
vocab_files_names = VOCAB_FILES_NAMES
diff --git a/src/transformers/models/squeezebert/tokenization_squeezebert_fast.py b/src/transformers/models/squeezebert/tokenization_squeezebert_fast.py
index ae1afb282e..58708030f9 100644
--- a/src/transformers/models/squeezebert/tokenization_squeezebert_fast.py
+++ b/src/transformers/models/squeezebert/tokenization_squeezebert_fast.py
@@ -54,11 +54,10 @@ class SqueezeBertTokenizerFast(BertTokenizerFast):
r"""
Constructs a "Fast" SqueezeBert tokenizer (backed by HuggingFace's *tokenizers* library).
- [`SqueezeBertTokenizerFast`] is identical to [`BertTokenizerFast`] and runs
- end-to-end tokenization: punctuation splitting + wordpiece.
+ [`SqueezeBertTokenizerFast`] is identical to [`BertTokenizerFast`] and runs end-to-end tokenization: punctuation
+ splitting + wordpiece.
- Refer to superclass [`BertTokenizerFast`] for usage examples and documentation concerning
- parameters.
+ Refer to superclass [`BertTokenizerFast`] for usage examples and documentation concerning parameters.
"""
vocab_files_names = VOCAB_FILES_NAMES
diff --git a/src/transformers/models/t5/configuration_t5.py b/src/transformers/models/t5/configuration_t5.py
index 557ff5e187..372417a9fc 100644
--- a/src/transformers/models/t5/configuration_t5.py
+++ b/src/transformers/models/t5/configuration_t5.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" T5 model configuration """
+""" T5 model configuration"""
from typing import Mapping
from ...configuration_utils import PretrainedConfig
@@ -33,13 +33,13 @@ T5_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class T5Config(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`T5Model`] or a
- [`TFT5Model`]. It is used to instantiate a T5 model according to the specified arguments,
- defining the model architecture. Instantiating a configuration with the defaults will yield a similar configuration
- to that of the T5 [t5-small](https://huggingface.co/t5-small) architecture.
+ This is the configuration class to store the configuration of a [`T5Model`] or a [`TFT5Model`]. It is used to
+ instantiate a T5 model according to the specified arguments, defining the model architecture. Instantiating a
+ configuration with the defaults will yield a similar configuration to that of the T5
+ [t5-small](https://huggingface.co/t5-small) architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Arguments:
vocab_size (`int`, *optional*, defaults to 32128):
@@ -48,14 +48,14 @@ class T5Config(PretrainedConfig):
d_model (`int`, *optional*, defaults to 512):
Size of the encoder layers and the pooler layer.
d_kv (`int`, *optional*, defaults to 64):
- Size of the key, query, value projections per attention head. `d_kv` has to be equal to `d_model // num_heads`.
+ Size of the key, query, value projections per attention head. `d_kv` has to be equal to `d_model //
+ num_heads`.
d_ff (`int`, *optional*, defaults to 2048):
Size of the intermediate feed forward layer in each `T5Block`.
num_layers (`int`, *optional*, defaults to 6):
Number of hidden layers in the Transformer encoder.
num_decoder_layers (`int`, *optional*):
- Number of hidden layers in the Transformer decoder. Will use the same value as `num_layers` if not
- set.
+ Number of hidden layers in the Transformer decoder. Will use the same value as `num_layers` if not set.
num_heads (`int`, *optional*, defaults to 8):
Number of attention heads for each attention layer in the Transformer encoder.
relative_attention_num_buckets (`int`, *optional*, defaults to 32):
@@ -68,8 +68,8 @@ class T5Config(PretrainedConfig):
A factor for initializing all weight matrices (should be kept to 1, used internally for initialization
testing).
feed_forward_proj (`string`, *optional*, defaults to `"relu"`):
- Type of feed forward layer to be used. Should be one of `"relu"` or `"gated-gelu"`. T5v1.1 uses
- the `"gated-gelu"` feed forward projection. Original T5 uses `"relu"`.
+ Type of feed forward layer to be used. Should be one of `"relu"` or `"gated-gelu"`. T5v1.1 uses the
+ `"gated-gelu"` feed forward projection. Original T5 uses `"relu"`.
use_cache (`bool`, *optional*, defaults to `True`):
Whether or not the model should return the last key/values attentions (not used by all models).
"""
diff --git a/src/transformers/models/t5/modeling_flax_t5.py b/src/transformers/models/t5/modeling_flax_t5.py
index 8d6cae04e4..4d3aa3f397 100644
--- a/src/transformers/models/t5/modeling_flax_t5.py
+++ b/src/transformers/models/t5/modeling_flax_t5.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" Flax T5 model. """
+""" Flax T5 model."""
import copy
@@ -788,9 +788,8 @@ T5_ENCODE_INPUTS_DOCSTRING = r"""
Indices of input sequence tokens in the vocabulary. T5 is a model with relative position embeddings so you
should be able to pad the inputs on both the right and the left.
- Indices can be obtained using [`T5Tokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- detail.
+ Indices can be obtained using [`T5Tokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for detail.
To know more on how to prepare `input_ids` for pretraining take a look a [T5 Training](./t5#training).
attention_mask (`jnp.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
@@ -815,18 +814,16 @@ T5_DECODE_INPUTS_DOCSTRING = r"""
decoder_input_ids (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`):
Indices of decoder input sequence tokens in the vocabulary.
- Indices can be obtained using [`T5Tokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`T5Tokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are decoder input IDs?](../glossary#decoder-input-ids)
For training, `decoder_input_ids` should be provided.
encoder_outputs (`tuple(tuple(jnp.ndarray)`):
- Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*:
- `attentions`) `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`,
- *optional*) is a sequence of hidden-states at the output of the last layer of the encoder. Used in the
- cross-attention of the decoder.
+ Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`)
+ `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of
+ hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder.
encoder_attention_mask (`jnp.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
@@ -835,10 +832,11 @@ T5_DECODE_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
decoder_attention_mask (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`, *optional*):
- Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will
- also be used by default.
+ Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
+ be used by default.
- If you want to change padding behavior, you should modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy.
+ If you want to change padding behavior, you should modify to your needs. See diagram 1 in [the
+ paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy.
past_key_values (`Dict[str, np.ndarray]`, *optional*, returned by `init_cache` or when passing previous `past_key_values`):
Dictionary of pre-computed hidden-states (key and values in the attention blocks) that can be used for fast
auto-regressive decoding. Pre-computed key and value hidden-states are of shape *[batch_size, max_length]*.
@@ -859,9 +857,8 @@ T5_INPUTS_DOCSTRING = r"""
Indices of input sequence tokens in the vocabulary. T5 is a model with relative position embeddings so you
should be able to pad the inputs on both the right and the left.
- Indices can be obtained using [`T5Tokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- detail.
+ Indices can be obtained using [`T5Tokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for detail.
[What are input IDs?](../glossary#input-ids)
@@ -876,31 +873,29 @@ T5_INPUTS_DOCSTRING = r"""
decoder_input_ids (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`, *optional*):
Indices of decoder input sequence tokens in the vocabulary.
- Indices can be obtained using [`T5Tokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`T5Tokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are decoder input IDs?](../glossary#decoder-input-ids)
- T5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If
- `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
- `past_key_values`).
+ T5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values`
+ is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`).
- To know more on how to prepare `decoder_input_ids` for pretraining take a look at [T5 Training](./t5#training).
+ To know more on how to prepare `decoder_input_ids` for pretraining take a look at [T5
+ Training](./t5#training).
decoder_attention_mask (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`, *optional*):
- Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will
- also be used by default.
+ Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
+ be used by default.
encoder_outputs (`tuple(tuple(jnp.ndarray)`, *optional*):
- Tuple consists of (`last_hidden_state`, `optional`: *hidden_states*, `optional`:
- *attentions*) `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)` is a
- sequence of hidden states at the output of the last layer of the encoder. Used in the cross-attention of
- the decoder.
+ Tuple consists of (`last_hidden_state`, `optional`: *hidden_states*, `optional`: *attentions*)
+ `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)` is a sequence of hidden states at
+ the output of the last layer of the encoder. Used in the cross-attention of the decoder.
past_key_values (`tuple(tuple(jnp.ndarray))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ `decoder_input_ids` of shape `(batch_size, sequence_length)`.
output_attentions (`bool`, *optional*):
@@ -1012,9 +1007,10 @@ class FlaxT5PreTrainedModel(FlaxPreTrainedModel):
maximum possible length for auto-regressive decoding. Defines the sequence length of the initialized
cache.
encoder_outputs (`Union[FlaxBaseModelOutput, tuple(tuple(jnp.ndarray)]`):
- `encoder_outputs` consists of (`last_hidden_state`, *optional*: `hidden_states`,
- *optional*: `attentions`). `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of hidden-states at the output of the last layer of the
- encoder. Used in the cross-attention of the decoder.
+ `encoder_outputs` consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*:
+ `attentions`). `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*)
+ is a sequence of hidden-states at the output of the last layer of the encoder. Used in the
+ cross-attention of the decoder.
"""
# init input variables to retrieve cache
decoder_input_ids = jnp.ones((batch_size, max_length), dtype="i4")
@@ -1201,16 +1197,18 @@ class FlaxT5PreTrainedModel(FlaxPreTrainedModel):
T5_START_DOCSTRING = r"""
- The T5 model was proposed in [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/abs/1910.10683) by Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang,
- Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu. It's an encoder decoder transformer pre-trained in a text-to-text
- denoising generative setting.
+ The T5 model was proposed in [Exploring the Limits of Transfer Learning with a Unified Text-to-Text
+ Transformer](https://arxiv.org/abs/1910.10683) by Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan
+ Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu. It's an encoder decoder transformer pre-trained in a
+ text-to-text denoising generative setting.
- This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading or saving, resizing the input
- embeddings, pruning heads etc.)
+ This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a Flax Linen [flax.nn.Module](https://flax.readthedocs.io/en/latest/_autosummary/flax.nn.module.html) subclass. Use it as a regular Flax
- Module and refer to the Flax documentation for all matter related to general usage and behavior.
+ This model is also a Flax Linen
+ [flax.nn.Module](https://flax.readthedocs.io/en/latest/_autosummary/flax.nn.module.html) subclass. Use it as a
+ regular Flax Module and refer to the Flax documentation for all matter related to general usage and behavior.
Finally, this model supports inherent JAX features such as:
@@ -1222,11 +1220,10 @@ T5_START_DOCSTRING = r"""
Parameters:
config ([`T5Config`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the
- model weights.
+ configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the model weights.
dtype (`jax.numpy.dtype`, *optional*, defaults to `jax.numpy.float32`):
- The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on
- GPUs) and `jax.numpy.bfloat16` (on TPUs).
+ The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on GPUs) and
+ `jax.numpy.bfloat16` (on TPUs).
This can be used to enable mixed-precision training or half-precision inference on GPUs or TPUs. If
specified all the computation will be performed with the given `dtype`.
@@ -1234,8 +1231,8 @@ T5_START_DOCSTRING = r"""
**Note that this only specifies the dtype of the computation and does not influence the dtype of model
parameters.**
- If you wish to change the dtype of the model parameters, see
- [`~FlaxPreTrainedModel.to_fp16`] and [`~FlaxPreTrainedModel.to_bf16`].
+ If you wish to change the dtype of the model parameters, see [`~FlaxPreTrainedModel.to_fp16`] and
+ [`~FlaxPreTrainedModel.to_bf16`].
"""
@@ -1353,7 +1350,7 @@ overwrite_call_docstring(FlaxT5Model, T5_INPUTS_DOCSTRING + FLAX_T5_MODEL_DOCSTR
append_replace_return_docstrings(FlaxT5Model, output_type=FlaxSeq2SeqLMOutput, config_class=_CONFIG_FOR_DOC)
-@add_start_docstrings("""T5 Model with a `language modeling` head on top. """, T5_START_DOCSTRING)
+@add_start_docstrings("""T5 Model with a `language modeling` head on top.""", T5_START_DOCSTRING)
class FlaxT5ForConditionalGenerationModule(nn.Module):
config: T5Config
dtype: jnp.dtype = jnp.float32 # the dtype of the computation
diff --git a/src/transformers/models/t5/modeling_t5.py b/src/transformers/models/t5/modeling_t5.py
index 3f34de5476..ab3b7f2961 100644
--- a/src/transformers/models/t5/modeling_t5.py
+++ b/src/transformers/models/t5/modeling_t5.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" PyTorch T5 model. """
+""" PyTorch T5 model."""
import copy
@@ -1078,23 +1078,23 @@ class T5Stack(T5PreTrainedModel):
T5_START_DOCSTRING = r"""
- The T5 model was proposed in [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/abs/1910.10683) by Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang,
- Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu. It's an encoder decoder transformer pre-trained in a text-to-text
- denoising generative setting.
+ The T5 model was proposed in [Exploring the Limits of Transfer Learning with a Unified Text-to-Text
+ Transformer](https://arxiv.org/abs/1910.10683) by Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan
+ Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu. It's an encoder decoder transformer pre-trained in a
+ text-to-text denoising generative setting.
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
Parameters:
config ([`T5Config`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
T5_INPUTS_DOCSTRING = r"""
@@ -1103,9 +1103,8 @@ T5_INPUTS_DOCSTRING = r"""
Indices of input sequence tokens in the vocabulary. T5 is a model with relative position embeddings so you
should be able to pad the inputs on both the right and the left.
- Indices can be obtained using [`T5Tokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- detail.
+ Indices can be obtained using [`T5Tokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for detail.
[What are input IDs?](../glossary#input-ids)
@@ -1120,28 +1119,29 @@ T5_INPUTS_DOCSTRING = r"""
decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
Indices of decoder input sequence tokens in the vocabulary.
- Indices can be obtained using [`T5Tokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`T5Tokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are decoder input IDs?](../glossary#decoder-input-ids)
- T5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If
- `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
- `past_key_values`).
+ T5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values`
+ is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`).
- To know more on how to prepare `decoder_input_ids` for pretraining take a look at [T5 Training](./t5#training).
+ To know more on how to prepare `decoder_input_ids` for pretraining take a look at [T5
+ Training](./t5#training).
decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
- Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will
- also be used by default.
+ Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
+ be used by default.
head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
- Mask to nullify selected heads of the self-attention modules in the encoder. Mask values selected in `[0, 1]`:
+ Mask to nullify selected heads of the self-attention modules in the encoder. Mask values selected in `[0,
+ 1]`:
- 1 indicates the head is **not masked**,
- 0 indicates the head is **masked**.
decoder_head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
- Mask to nullify selected heads of the self-attention modules in the decoder. Mask values selected in `[0, 1]`:
+ Mask to nullify selected heads of the self-attention modules in the decoder. Mask values selected in `[0,
+ 1]`:
- 1 indicates the head is **not masked**,
- 0 indicates the head is **masked**.
@@ -1154,32 +1154,31 @@ T5_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*):
- Tuple consists of (`last_hidden_state`, `optional`: *hidden_states*, `optional`:
- *attentions*) `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)` is a
- sequence of hidden states at the output of the last layer of the encoder. Used in the cross-attention of
- the decoder.
+ Tuple consists of (`last_hidden_state`, `optional`: *hidden_states*, `optional`: *attentions*)
+ `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)` is a sequence of hidden states at
+ the output of the last layer of the encoder. Used in the cross-attention of the decoder.
past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ `decoder_input_ids` of shape `(batch_size, sequence_length)`.
inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*):
Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded
- representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds`
- have to be input (see `past_key_values`). This is useful if you want more control over how to convert
+ representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be
+ input (see `past_key_values`). This is useful if you want more control over how to convert
`decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix.
- If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds`
- takes the value of `inputs_embeds`.
+ If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds` takes the value
+ of `inputs_embeds`.
use_cache (`bool`, *optional*):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`).
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`).
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
@@ -1197,9 +1196,8 @@ T5_ENCODER_INPUTS_DOCSTRING = r"""
Indices of input sequence tokens in the vocabulary. T5 is a model with relative position embeddings so you
should be able to pad the inputs on both the right and the left.
- Indices can be obtained using [`T5Tokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- detail.
+ Indices can be obtained using [`T5Tokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for detail.
To know more on how to prepare `input_ids` for pretraining take a look a [T5 Training](./t5#training).
attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -1216,9 +1214,9 @@ T5_ENCODER_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -1428,7 +1426,7 @@ class T5Model(T5PreTrainedModel):
)
-@add_start_docstrings("""T5 Model with a `language modeling` head on top. """, T5_START_DOCSTRING)
+@add_start_docstrings("""T5 Model with a `language modeling` head on top.""", T5_START_DOCSTRING)
class T5ForConditionalGeneration(T5PreTrainedModel):
_keys_to_ignore_on_load_missing = [
r"encoder\.embed_tokens\.weight",
@@ -1533,7 +1531,8 @@ class T5ForConditionalGeneration(T5PreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[-100, 0, ..., config.vocab_size - 1]`. All labels set to `-100` are ignored (masked), the loss is only computed for
+ Labels for computing the sequence classification/regression loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size - 1]`. All labels set to `-100` are ignored (masked), the loss is only computed for
labels in `[0, ..., config.vocab_size]`
Returns:
diff --git a/src/transformers/models/t5/modeling_tf_t5.py b/src/transformers/models/t5/modeling_tf_t5.py
index 36654c68ff..c779384ff8 100644
--- a/src/transformers/models/t5/modeling_tf_t5.py
+++ b/src/transformers/models/t5/modeling_tf_t5.py
@@ -13,7 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" TF 2.0 T5 model. """
+""" TF 2.0 T5 model."""
import copy
import itertools
@@ -946,17 +946,18 @@ class TFT5PreTrainedModel(TFPreTrainedModel):
T5_START_DOCSTRING = r"""
- The T5 model was proposed in [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/abs/1910.10683) by Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang,
- Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu. It's an encoder decoder transformer pre-trained in a text-to-text
- denoising generative setting.
+ The T5 model was proposed in [Exploring the Limits of Transfer Learning with a Unified Text-to-Text
+ Transformer](https://arxiv.org/abs/1910.10683) by Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan
+ Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu. It's an encoder decoder transformer pre-trained in a
+ text-to-text denoising generative setting.
- This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading or saving, resizing the input
- embeddings, pruning heads etc.)
+ This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use
- it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage
- and behavior.
+ This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it
+ as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and
+ behavior.
@@ -965,11 +966,11 @@ T5_START_DOCSTRING = r"""
- having all inputs as keyword arguments (like PyTorch models), or
- having all inputs as a list, tuple or dict in the first positional arguments.
- This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all
- the tensors in the first argument of the model call function: `model(inputs)`.
+ This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all the
+ tensors in the first argument of the model call function: `model(inputs)`.
- If you choose this second option, there are three possibilities you can use to gather all the input Tensors in
- the first positional argument :
+ If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the
+ first positional argument :
- a single Tensor with `input_ids` only and nothing else: `model(inputs_ids)`
- a list of varying length with one or several input Tensors IN THE ORDER given in the docstring:
@@ -982,8 +983,7 @@ T5_START_DOCSTRING = r"""
Parameters:
config ([`T5Config`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
T5_INPUTS_DOCSTRING = r"""
@@ -992,19 +992,19 @@ T5_INPUTS_DOCSTRING = r"""
Indices of input sequence tokens in the vocabulary. T5 is a model with relative position embeddings so you
should be able to pad the inputs on the right or the left.
- Indices can be obtained using [`BertTokenizer`]. See
- [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for
- details.
+ Indices can be obtained using [`BertTokenizer`]. See [`PreTrainedTokenizer.__call__`] and
+ [`PreTrainedTokenizer.encode`] for details.
[What are input IDs?](../glossary#input-ids)
To know more on how to prepare `inputs` for pretraining take a look at [T5 Training](./t5#training).
decoder_input_ids (`tf.Tensor` of shape `(batch_size, target_sequence_length)`, *optional*):
Provide for sequence to sequence training. T5 uses the `pad_token_id` as the starting token for
- `decoder_input_ids` generation. If `past_key_values` is used, optionally only the last
- `decoder_input_ids` have to be input (see `past_key_values`).
+ `decoder_input_ids` generation. If `past_key_values` is used, optionally only the last `decoder_input_ids`
+ have to be input (see `past_key_values`).
- To know more on how to prepare `decoder_input_ids` for pretraining take a look at [T5 Training](./t5#training).
+ To know more on how to prepare `decoder_input_ids` for pretraining take a look at [T5
+ Training](./t5#training).
attention_mask (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
@@ -1013,46 +1013,47 @@ T5_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
decoder_attention_mask (`tf.Tensor` of shape `(batch_size, target_sequence_length)`, *optional*):
- Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will
- also be used by default.
+ Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
+ be used by default.
head_mask: (`tf.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
- Mask to nullify selected heads of the self-attention modules in the encoder. Mask values selected in `[0, 1]`:
+ Mask to nullify selected heads of the self-attention modules in the encoder. Mask values selected in `[0,
+ 1]`:
- 1 indicates the head is **not masked**,
- 0 indicates the head is **masked**.
decoder_head_mask: (`tf.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
- Mask to nullify selected heads of the self-attention modules in the decoder. Mask values selected in `[0, 1]`:
+ Mask to nullify selected heads of the self-attention modules in the decoder. Mask values selected in `[0,
+ 1]`:
- 1 indicates the head is **not masked**,
- 0 indicates the head is **masked**.
encoder_outputs (`tuple(tuple(tf.FloatTensor)`, *optional*):
- Tuple consists of (`last_hidden_state`, `optional`: *hidden_states*, `optional`:
- *attentions*) `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)` is a
- sequence of hidden states at the output of the last layer of the encoder. Used in the cross-attention of
- the decoder.
+ Tuple consists of (`last_hidden_state`, `optional`: *hidden_states*, `optional`: *attentions*)
+ `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)` is a sequence of hidden states at
+ the output of the last layer of the encoder. Used in the cross-attention of the decoder.
past_key_values (`tuple(tuple(tf.Tensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ `decoder_input_ids` of shape `(batch_size, sequence_length)`.
inputs_embeds (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
decoder_inputs_embeds (`tf.Tensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*):
Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded
- representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds`
- have to be input (see `past_key_values`). This is useful if you want more control over how to convert
+ representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be
+ input (see `past_key_values`). This is useful if you want more control over how to convert
`decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix.
- If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds`
- takes the value of `inputs_embeds`.
+ If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds` takes the value
+ of `inputs_embeds`.
use_cache (`bool`, *optional*, defaults to `True`):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`).
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`).
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the
@@ -1062,8 +1063,8 @@ T5_INPUTS_DOCSTRING = r"""
more detail. This argument can be used only in eager mode, in graph mode the value in the config will be
used instead.
return_dict (`bool`, *optional*):
- Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This
- argument can be used in eager mode, in graph mode the value will always be set to True.
+ Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This argument can be used
+ in eager mode, in graph mode the value will always be set to True.
training (`bool`, *optional*, defaults to `False`):
Whether or not to use the model in training mode (some modules like dropout modules have different
behaviors between training and evaluation).
@@ -1075,9 +1076,8 @@ T5_ENCODER_INPUTS_DOCSTRING = r"""
Indices of input sequence tokens in the vocabulary. T5 is a model with relative position embeddings so you
should be able to pad the inputs on the right or the left.
- Indices can be obtained using [`T5Tokenizer`]. See
- [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for
- details.
+ Indices can be obtained using [`T5Tokenizer`]. See [`PreTrainedTokenizer.__call__`] and
+ [`PreTrainedTokenizer.encode`] for details.
To know more on how to prepare `inputs` for pre-training take a look at [T5 Training](./t5#training).
attention_mask (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -1088,9 +1088,9 @@ T5_ENCODER_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
inputs_embeds (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
head_mask: (`tf.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:
@@ -1285,7 +1285,7 @@ class TFT5Model(TFT5PreTrainedModel):
)
-@add_start_docstrings("""T5 Model with a `language modeling` head on top. """, T5_START_DOCSTRING)
+@add_start_docstrings("""T5 Model with a `language modeling` head on top.""", T5_START_DOCSTRING)
class TFT5ForConditionalGeneration(TFT5PreTrainedModel, TFCausalLanguageModelingLoss):
def __init__(self, config, *inputs, **kwargs):
super().__init__(config, *inputs, **kwargs)
@@ -1365,7 +1365,8 @@ class TFT5ForConditionalGeneration(TFT5PreTrainedModel, TFCausalLanguageModeling
):
r"""
labels (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the cross entropy classification loss. Indices should be in `[0, ..., config.vocab_size - 1]`.
+ Labels for computing the cross entropy classification loss. Indices should be in `[0, ...,
+ config.vocab_size - 1]`.
Returns:
diff --git a/src/transformers/models/t5/tokenization_t5.py b/src/transformers/models/t5/tokenization_t5.py
index 7b49310568..bb2fcb2e41 100644
--- a/src/transformers/models/t5/tokenization_t5.py
+++ b/src/transformers/models/t5/tokenization_t5.py
@@ -54,8 +54,8 @@ class T5Tokenizer(PreTrainedTokenizer):
"""
Construct a T5 tokenizer. Based on [SentencePiece](https://github.com/google/sentencepiece).
- This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods.
- Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
+ this superclass for more information regarding those methods.
Args:
vocab_file (`str`):
@@ -66,8 +66,8 @@ class T5Tokenizer(PreTrainedTokenizer):
- When building a sequence using special tokens, this is not the token that is used for the end of
- sequence. The token used is the `sep_token`.
+ When building a sequence using special tokens, this is not the token that is used for the end of sequence.
+ The token used is the `sep_token`.
@@ -80,11 +80,14 @@ class T5Tokenizer(PreTrainedTokenizer):
Add a number of extra ids added to the end of the vocabulary for use as sentinels. These tokens are
accessible as "" where "{%d}" is a number between 0 and extra_ids-1. Extra tokens are
indexed from the end of the vocabulary up to beginning ("" is the last token in the vocabulary
- like in T5 preprocessing see [here](https://github.com/google-research/text-to-text-transfer-transformer/blob/9fd7b14a769417be33bc6c850f9598764913c833/t5/data/preprocessors.py#L2117)).
+ like in T5 preprocessing see
+ [here](https://github.com/google-research/text-to-text-transfer-transformer/blob/9fd7b14a769417be33bc6c850f9598764913c833/t5/data/preprocessors.py#L2117)).
additional_special_tokens (`List[str]`, *optional*):
Additional special tokens used by the tokenizer.
sp_model_kwargs (`dict`, *optional*):
- Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things, to set:
+ Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for
+ SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things,
+ to set:
- `enable_sampling`: Enable subword regularization.
- `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout.
diff --git a/src/transformers/models/t5/tokenization_t5_fast.py b/src/transformers/models/t5/tokenization_t5_fast.py
index 88833b2daa..3749734245 100644
--- a/src/transformers/models/t5/tokenization_t5_fast.py
+++ b/src/transformers/models/t5/tokenization_t5_fast.py
@@ -62,10 +62,11 @@ PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
class T5TokenizerFast(PreTrainedTokenizerFast):
"""
- Construct a "fast" T5 tokenizer (backed by HuggingFace's *tokenizers* library). Based on [Unigram](https://huggingface.co/docs/tokenizers/python/latest/components.html?highlight=unigram#models).
+ Construct a "fast" T5 tokenizer (backed by HuggingFace's *tokenizers* library). Based on
+ [Unigram](https://huggingface.co/docs/tokenizers/python/latest/components.html?highlight=unigram#models).
- This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main
- methods. Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should
+ refer to this superclass for more information regarding those methods.
Args:
vocab_file (`str`):
@@ -76,8 +77,8 @@ class T5TokenizerFast(PreTrainedTokenizerFast):
- When building a sequence using special tokens, this is not the token that is used for the end of
- sequence. The token used is the `sep_token`.
+ When building a sequence using special tokens, this is not the token that is used for the end of sequence.
+ The token used is the `sep_token`.
@@ -90,7 +91,8 @@ class T5TokenizerFast(PreTrainedTokenizerFast):
Add a number of extra ids added to the end of the vocabulary for use as sentinels. These tokens are
accessible as "" where "{%d}" is a number between 0 and extra_ids-1. Extra tokens are
indexed from the end of the vocabulary up to beginning ("" is the last token in the vocabulary
- like in T5 preprocessing see [here](https://github.com/google-research/text-to-text-transfer-transformer/blob/9fd7b14a769417be33bc6c850f9598764913c833/t5/data/preprocessors.py#L2117)).
+ like in T5 preprocessing see
+ [here](https://github.com/google-research/text-to-text-transfer-transformer/blob/9fd7b14a769417be33bc6c850f9598764913c833/t5/data/preprocessors.py#L2117)).
additional_special_tokens (`List[str]`, *optional*):
Additional special tokens used by the tokenizer.
"""
diff --git a/src/transformers/models/tapas/configuration_tapas.py b/src/transformers/models/tapas/configuration_tapas.py
index 5c9c06dbc1..6eb78acc48 100644
--- a/src/transformers/models/tapas/configuration_tapas.py
+++ b/src/transformers/models/tapas/configuration_tapas.py
@@ -36,11 +36,11 @@ TAPAS_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class TapasConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`TapasModel`]. It is used to
- instantiate a TAPAS model according to the specified arguments, defining the model architecture. Instantiating a
- configuration with the defaults will yield a similar configuration to that of the TAPAS *tapas-base-finetuned-sqa*
- architecture. Configuration objects inherit from [`PreTrainedConfig`] and can be used to control
- the model outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ This is the configuration class to store the configuration of a [`TapasModel`]. It is used to instantiate a TAPAS
+ model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
+ defaults will yield a similar configuration to that of the TAPAS *tapas-base-finetuned-sqa* architecture.
+ Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Hyperparameters additional to BERT are taken from run_task_main.py and hparam_utils.py of the original
implementation. Original implementation available at https://github.com/google-research/tapas/tree/master.
@@ -58,8 +58,8 @@ class TapasConfig(PretrainedConfig):
intermediate_size (`int`, *optional*, defaults to 3072):
Dimensionality of the "intermediate" (often named feed-forward) layer in the Transformer encoder.
hidden_act (`str` or `Callable`, *optional*, defaults to `"gelu"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"swish"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"swish"` and `"gelu_new"` are supported.
hidden_dropout_prob (`float`, *optional*, defaults to 0.1):
The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
attention_probs_dropout_prob (`float`, *optional*, defaults to 0.1):
diff --git a/src/transformers/models/tapas/modeling_tapas.py b/src/transformers/models/tapas/modeling_tapas.py
index e9b87abafc..85b6ed1691 100644
--- a/src/transformers/models/tapas/modeling_tapas.py
+++ b/src/transformers/models/tapas/modeling_tapas.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-"""PyTorch TAPAS model. """
+"""PyTorch TAPAS model."""
import enum
@@ -116,12 +116,13 @@ class TableQuestionAnsweringOutput(ModelOutput):
logits_aggregation (`torch.FloatTensor`, *optional*, of shape `(batch_size, num_aggregation_labels)`):
Prediction scores of the aggregation head, for every aggregation operator.
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of
- each layer plus the initial embedding outputs.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer
+ plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the
- weighted average in the self-attention heads.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
+ the self-attention heads.
"""
loss: Optional[torch.FloatTensor] = None
@@ -791,27 +792,25 @@ class TapasPreTrainedModel(PreTrainedModel):
TAPAS_START_DOCSTRING = r"""
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its models (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its models (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
Parameters:
config ([`TapasConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
TAPAS_INPUTS_DOCSTRING = r"""
Args:
input_ids (`torch.LongTensor` of shape `({0})`):
- Indices of input sequence tokens in the vocabulary. Indices can be obtained using
- [`TapasTokenizer`]. See [`PreTrainedTokenizer.encode`] and
- [`PreTrainedTokenizer.__call__`] for details.
+ Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`TapasTokenizer`]. See
+ [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
@@ -822,23 +821,23 @@ TAPAS_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`torch.LongTensor` of shape `({0}, 7)`, *optional*):
- Token indices that encode tabular structure. Indices can be obtained using
- [`TapasTokenizer`]. See this class for more info.
+ Token indices that encode tabular structure. Indices can be obtained using [`TapasTokenizer`]. See this
+ class for more info.
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
Indices of positions of each input sequence tokens in the position embeddings. If
- `reset_position_index_per_cell` of [`TapasConfig`] is set to `True`, relative
- position embeddings will be used. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ `reset_position_index_per_cell` of [`TapasConfig`] is set to `True`, relative position embeddings will be
+ used. Selected in the range `[0, config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - 1
indicates the head is **not masked**, - 0 indicates the head is **masked**.
inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -856,8 +855,7 @@ TAPAS_INPUTS_DOCSTRING = r"""
)
class TapasModel(TapasPreTrainedModel):
"""
- This class is a small change compared to [`BertModel`], taking into account the additional token
- type ids.
+ This class is a small change compared to [`BertModel`], taking into account the additional token type ids.
The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of
cross-attention is added between the self-attention layers, following the architecture described in [Attention is
@@ -1006,7 +1004,7 @@ class TapasModel(TapasPreTrainedModel):
)
-@add_start_docstrings("""Tapas Model with a `language modeling` head on top. """, TAPAS_START_DOCSTRING)
+@add_start_docstrings("""Tapas Model with a `language modeling` head on top.""", TAPAS_START_DOCSTRING)
class TapasForMaskedLM(TapasPreTrainedModel):
config_class = TapasConfig
base_model_prefix = "tapas"
@@ -1046,8 +1044,9 @@ class TapasForMaskedLM(TapasPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
Returns:
@@ -1184,18 +1183,18 @@ class TapasForQuestionAnswering(TapasPreTrainedModel):
aggregation_labels (`torch.LongTensor` of shape `(batch_size, )`, *optional*):
Aggregation function index for every example in the batch for computing the aggregation loss. Indices
- should be in `[0, ..., config.num_aggregation_labels - 1]`. Only required in case of strong
- supervision for aggregation (WikiSQL-supervised).
+ should be in `[0, ..., config.num_aggregation_labels - 1]`. Only required in case of strong supervision for
+ aggregation (WikiSQL-supervised).
float_answer (`torch.FloatTensor` of shape `(batch_size, )`, *optional*):
Float answer for every example in the batch. Set to *float('nan')* for cell selection questions. Only
required in case of weak supervision (WTQ) to calculate the aggregate mask and regression loss.
numeric_values (`torch.FloatTensor` of shape `(batch_size, seq_length)`, *optional*):
Numeric values of every token, NaN for tokens which are not numeric values. Can be obtained using
- [`TapasTokenizer`]. Only required in case of weak supervision for aggregation (WTQ) to
- calculate the regression loss.
+ [`TapasTokenizer`]. Only required in case of weak supervision for aggregation (WTQ) to calculate the
+ regression loss.
numeric_values_scale (`torch.FloatTensor` of shape `(batch_size, seq_length)`, *optional*):
- Scale of the numeric values of every token. Can be obtained using [`TapasTokenizer`].
- Only required in case of weak supervision for aggregation (WTQ) to calculate the regression loss.
+ Scale of the numeric values of every token. Can be obtained using [`TapasTokenizer`]. Only required in case
+ of weak supervision for aggregation (WTQ) to calculate the regression loss.
Returns:
@@ -1488,8 +1487,9 @@ class TapasForSequenceClassification(TapasPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). Note: this is called
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy). Note: this is called
"classification_class_index" in the original implementation.
Returns:
@@ -1945,8 +1945,8 @@ def compute_column_logits(
Whether to allow not to select any column
Returns:
- column_logits (`torch.FloatTensor`of shape `(batch_size, max_num_cols)`): Tensor containing the
- column logits for every example in the batch.
+ column_logits (`torch.FloatTensor`of shape `(batch_size, max_num_cols)`): Tensor containing the column logits
+ for every example in the batch.
"""
# First, compute the token logits (batch_size, seq_len) - without temperature
@@ -1997,10 +1997,10 @@ def _single_column_cell_selection_loss(token_logits, column_logits, labels, cell
Mask for cells that exist in the table (i.e. that are not padding).
Returns:
- selection_loss_per_example (`torch.FloatTensor` of shape `(batch_size,)`): Loss for each example.
- logits (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): New logits which are only
- allowed to select cells in a single column. Logits outside of the most likely column according to
- *column_logits* will be set to a very low value (such that the probabilities are 0).
+ selection_loss_per_example (`torch.FloatTensor` of shape `(batch_size,)`): Loss for each example. logits
+ (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): New logits which are only allowed to select
+ cells in a single column. Logits outside of the most likely column according to *column_logits* will be set to
+ a very low value (such that the probabilities are 0).
"""
# Part 1: column loss
@@ -2127,8 +2127,8 @@ def _calculate_aggregate_mask(answer, pooled_output, cell_selection_preference,
Labels per token. aggregation_classifier (`torch.nn.Linear`): Aggregation head
Returns:
- aggregate_mask (`torch.FloatTensor` of shape `(batch_size,)`): A mask set to 1 for examples that
- should use aggregation functions.
+ aggregate_mask (`torch.FloatTensor` of shape `(batch_size,)`): A mask set to 1 for examples that should use
+ aggregation functions.
"""
# torch.FloatTensor(batch_size,)
aggregate_mask_init = torch.logical_not(torch.isnan(answer)).type(torch.FloatTensor).to(answer.device)
@@ -2179,8 +2179,8 @@ def _calculate_aggregation_loss_known(
The number of aggregation operators to predict.
Returns:
- aggregation_loss_known (`torch.FloatTensor` of shape `(batch_size,)`): Aggregation loss (when its
- type is known during training) per example.
+ aggregation_loss_known (`torch.FloatTensor` of shape `(batch_size,)`): Aggregation loss (when its type is known
+ during training) per example.
"""
if use_answer_as_supervision:
# Prepare "no aggregation" targets for cell selection examples.
@@ -2213,8 +2213,8 @@ def _calculate_aggregation_loss_unknown(logits_aggregation, aggregate_mask):
A mask set to 1 for examples that should use aggregation functions
Returns:
- aggregation_loss_unknown (`torch.FloatTensor` of shape `(batch_size,)`): Aggregation loss (in case of
- answer supervision) per example.
+ aggregation_loss_unknown (`torch.FloatTensor` of shape `(batch_size,)`): Aggregation loss (in case of answer
+ supervision) per example.
"""
dist_aggregation = torch.distributions.categorical.Categorical(logits=logits_aggregation)
# Index 0 corresponds to "no aggregation".
@@ -2389,9 +2389,9 @@ def _calculate_regression_loss(
Model configuration class with all the parameters of the model
Returns:
- per_example_answer_loss_scaled (`torch.FloatTensor` of shape `(batch_size,)`): Scales answer loss for
- each example in the batch. large_answer_loss_mask (`torch.FloatTensor` of shape `(batch_size,)`): A
- mask which is 1 for examples for which their answer loss is larger than the answer_loss_cutoff.
+ per_example_answer_loss_scaled (`torch.FloatTensor` of shape `(batch_size,)`): Scales answer loss for each
+ example in the batch. large_answer_loss_mask (`torch.FloatTensor` of shape `(batch_size,)`): A mask which is 1
+ for examples for which their answer loss is larger than the answer_loss_cutoff.
"""
# float32 (batch_size,)
expected_result = _calculate_expected_result(
diff --git a/src/transformers/models/tapas/modeling_tf_tapas.py b/src/transformers/models/tapas/modeling_tf_tapas.py
index 989804d395..27ab31fff7 100644
--- a/src/transformers/models/tapas/modeling_tf_tapas.py
+++ b/src/transformers/models/tapas/modeling_tf_tapas.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-"""TF 2.0 TAPAS model. """
+"""TF 2.0 TAPAS model."""
import enum
import math
@@ -124,11 +124,12 @@ class TFTableQuestionAnsweringOutput(ModelOutput):
logits_aggregation (`tf.Tensor`, *optional*, of shape `(batch_size, num_aggregation_labels)`):
Prediction scores of the aggregation head, for every aggregation operator.
hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each
- layer plus the initial embedding outputs.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer plus
+ the initial embedding outputs.
attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in
the self-attention heads.
"""
@@ -879,13 +880,13 @@ class TFTapasPreTrainedModel(TFPreTrainedModel):
TAPAS_START_DOCSTRING = r"""
- This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading or saving, resizing the input
- embeddings, pruning heads etc.)
+ This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use
- it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage
- and behavior.
+ This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it
+ as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and
+ behavior.
@@ -894,11 +895,11 @@ TAPAS_START_DOCSTRING = r"""
- having all inputs as keyword arguments (like PyTorch models), or
- having all inputs as a list, tuple or dict in the first positional arguments.
- This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all
- the tensors in the first argument of the model call function: `model(inputs)`.
+ This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all the
+ tensors in the first argument of the model call function: `model(inputs)`.
- If you choose this second option, there are three possibilities you can use to gather all the input Tensors in
- the first positional argument :
+ If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the
+ first positional argument :
- a single Tensor with `input_ids` only and nothing else: `model(inputs_ids)`
- a list of varying length with one or several input Tensors IN THE ORDER given in the docstring:
@@ -911,8 +912,7 @@ TAPAS_START_DOCSTRING = r"""
Parameters:
config ([`TapasConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
TAPAS_INPUTS_DOCSTRING = r"""
@@ -920,9 +920,8 @@ TAPAS_INPUTS_DOCSTRING = r"""
input_ids (`np.ndarray`, `tf.Tensor`, `List[tf.Tensor]` ``Dict[str, tf.Tensor]` or `Dict[str, np.ndarray]` and each example must have the shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`TapasTokenizer`]. See
- [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for
- details.
+ Indices can be obtained using [`TapasTokenizer`]. See [`PreTrainedTokenizer.__call__`] and
+ [`PreTrainedTokenizer.encode`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`np.ndarray` or `tf.Tensor` of shape `({0})`, *optional*):
@@ -933,14 +932,14 @@ TAPAS_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`np.ndarray` or `tf.Tensor` of shape `({0}, 7)`, *optional*):
- Token indices that encode tabular structure. Indices can be obtained using
- [`TapasTokenizer`]. See this class for more info.
+ Token indices that encode tabular structure. Indices can be obtained using [`TapasTokenizer`]. See this
+ class for more info.
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`np.ndarray` or `tf.Tensor` of shape `({0})`, *optional*):
Indices of positions of each input sequence tokens in the position embeddings. If
- `reset_position_index_per_cell` of [`TapasConfig`] is set to `True`, relative
- position embeddings will be used. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ `reset_position_index_per_cell` of [`TapasConfig`] is set to `True`, relative position embeddings will be
+ used. Selected in the range `[0, config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
head_mask (`np.ndarray` or `tf.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
@@ -950,9 +949,9 @@ TAPAS_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
inputs_embeds (`np.ndarray` or `tf.Tensor` of shape `({0}, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the
@@ -962,8 +961,8 @@ TAPAS_INPUTS_DOCSTRING = r"""
more detail. This argument can be used only in eager mode, in graph mode the value in the config will be
used instead.
return_dict (`bool`, *optional*):
- Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This
- argument can be used in eager mode, in graph mode the value will always be set to True.
+ Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This argument can be used
+ in eager mode, in graph mode the value will always be set to True.
training (`bool`, *optional*, defaults to `False``):
Whether or not to use the model in training mode (some modules like dropout modules have different
behaviors between training and evaluation).
@@ -1062,7 +1061,7 @@ class TFTapasModel(TFTapasPreTrainedModel):
)
-@add_start_docstrings("""Tapas Model with a `language modeling` head on top. """, TAPAS_START_DOCSTRING)
+@add_start_docstrings("""Tapas Model with a `language modeling` head on top.""", TAPAS_START_DOCSTRING)
class TFTapasForMaskedLM(TFTapasPreTrainedModel, TFMaskedLanguageModelingLoss):
def __init__(self, config: TapasConfig, *inputs, **kwargs):
super().__init__(config, *inputs, **kwargs)
@@ -1098,8 +1097,9 @@ class TFTapasForMaskedLM(TFTapasPreTrainedModel, TFMaskedLanguageModelingLoss):
) -> Union[TFMaskedLMOutput, Tuple[tf.Tensor]]:
r"""
labels (`tf.Tensor` or `np.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
Returns:
@@ -1246,8 +1246,8 @@ class TFTapasComputeColumnLogits(tf.keras.layers.Layer):
Whether to allow not to select any column
Returns:
- column_logits (`tf.Tensor`of shape `(batch_size, max_num_cols)`): Tensor containing the column
- logits for every example in the batch.
+ column_logits (`tf.Tensor`of shape `(batch_size, max_num_cols)`): Tensor containing the column logits for
+ every example in the batch.
"""
# First, compute the token logits (batch_size, seq_len) - without temperature
@@ -1338,18 +1338,18 @@ class TFTapasForQuestionAnswering(TFTapasPreTrainedModel):
aggregation_labels (`tf.Tensor` of shape `(batch_size, )`, *optional*):
Aggregation function index for every example in the batch for computing the aggregation loss. Indices
- should be in `[0, ..., config.num_aggregation_labels - 1]`. Only required in case of strong
- supervision for aggregation (WikiSQL-supervised).
+ should be in `[0, ..., config.num_aggregation_labels - 1]`. Only required in case of strong supervision for
+ aggregation (WikiSQL-supervised).
float_answer (`tf.Tensor` of shape `(batch_size, )`, *optional*):
Float answer for every example in the batch. Set to *float('nan')* for cell selection questions. Only
required in case of weak supervision (WTQ) to calculate the aggregate mask and regression loss.
numeric_values (`tf.Tensor` of shape `(batch_size, seq_length)`, *optional*):
Numeric values of every token, NaN for tokens which are not numeric values. Can be obtained using
- [`TapasTokenizer`]. Only required in case of weak supervision for aggregation (WTQ) to
- calculate the regression loss.
+ [`TapasTokenizer`]. Only required in case of weak supervision for aggregation (WTQ) to calculate the
+ regression loss.
numeric_values_scale (`tf.Tensor` of shape `(batch_size, seq_length)`, *optional*):
- Scale of the numeric values of every token. Can be obtained using [`TapasTokenizer`].
- Only required in case of weak supervision for aggregation (WTQ) to calculate the regression loss.
+ Scale of the numeric values of every token. Can be obtained using [`TapasTokenizer`]. Only required in case
+ of weak supervision for aggregation (WTQ) to calculate the regression loss.
Returns:
@@ -1667,8 +1667,9 @@ class TFTapasForSequenceClassification(TFTapasPreTrainedModel, TFSequenceClassif
) -> Union[TFSequenceClassifierOutput, Tuple[tf.Tensor]]:
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). Note: this is called
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy). Note: this is called
"classification_class_index" in the original implementation.
Returns:
@@ -2023,10 +2024,10 @@ def _single_column_cell_selection_loss(token_logits, column_logits, labels, cell
Mask for cells that exist in the table (i.e. that are not padding).
Returns:
- selection_loss_per_example (`tf.Tensor` of shape `(batch_size,)`): Loss for each example. logits
- (`tf.Tensor` of shape `(batch_size, sequence_length)`): New logits which are only allowed to select
- cells in a single column. Logits outside of the most likely column according to *column_logits* will be set to
- a very low value (such that the probabilities are 0).
+ selection_loss_per_example (`tf.Tensor` of shape `(batch_size,)`): Loss for each example. logits (`tf.Tensor`
+ of shape `(batch_size, sequence_length)`): New logits which are only allowed to select cells in a single
+ column. Logits outside of the most likely column according to *column_logits* will be set to a very low value
+ (such that the probabilities are 0).
"""
# First find the column we should select. We use the column with maximum
# number of selected cells.
@@ -2097,8 +2098,8 @@ def _calculate_aggregate_mask(answer, pooled_output, cell_selection_preference,
Labels per token. aggregation_classifier (`torch.nn.Linear`): Aggregation head
Returns:
- aggregate_mask (`tf.Tensor` of shape `(batch_size,)`): A mask set to 1 for examples that should use
- aggregation functions.
+ aggregate_mask (`tf.Tensor` of shape `(batch_size,)`): A mask set to 1 for examples that should use aggregation
+ functions.
"""
# tf.Tensor(batch_size,)
aggregate_mask_init = tf.cast(tf.logical_not(tf.math.is_nan(answer)), tf.float32)
@@ -2142,8 +2143,8 @@ def _calculate_aggregation_loss_known(
The number of aggregation operators to predict.
Returns:
- aggregation_loss_known (`tf.Tensor` of shape `(batch_size,)`): Aggregation loss (when its type is
- known during training) per example.
+ aggregation_loss_known (`tf.Tensor` of shape `(batch_size,)`): Aggregation loss (when its type is known during
+ training) per example.
"""
if use_answer_as_supervision:
# Prepare "no aggregation" targets for cell selection examples.
@@ -2338,9 +2339,9 @@ def _calculate_regression_loss(
Model configuration class with all the parameters of the model
Returns:
- per_example_answer_loss_scaled (`tf.Tensor` of shape `(batch_size,)`): Scales answer loss for each
- example in the batch. large_answer_loss_mask (`tf.Tensor` of shape `(batch_size,)`): A mask which is
- 1 for examples for which their answer loss is larger than the answer_loss_cutoff.
+ per_example_answer_loss_scaled (`tf.Tensor` of shape `(batch_size,)`): Scales answer loss for each example in
+ the batch. large_answer_loss_mask (`tf.Tensor` of shape `(batch_size,)`): A mask which is 1 for examples for
+ which their answer loss is larger than the answer_loss_cutoff.
"""
# float32 (batch_size,)
expected_result = _calculate_expected_result(
diff --git a/src/transformers/models/tapas/tokenization_tapas.py b/src/transformers/models/tapas/tokenization_tapas.py
index 92117b7031..80d406ce89 100644
--- a/src/transformers/models/tapas/tokenization_tapas.py
+++ b/src/transformers/models/tapas/tokenization_tapas.py
@@ -89,8 +89,7 @@ PRETRAINED_INIT_CONFIGURATION = {name: {"do_lower_case": True} for name in PRETR
class TapasTruncationStrategy(ExplicitEnum):
"""
- Possible values for the `truncation` argument in [`~TapasTokenizer.__call__`]. Useful for
- tab-completion in an IDE.
+ Possible values for the `truncation` argument in [`~TapasTokenizer.__call__`]. Useful for tab-completion in an IDE.
"""
DROP_ROWS_TO_FIT = "drop_rows_to_fit"
@@ -151,26 +150,26 @@ TAPAS_ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING = r"""
padding (`bool`, `str` or [`~file_utils.PaddingStrategy`], *optional*, defaults to `False`):
Activates and controls padding. Accepts the following values:
- - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a
- single sequence if provided).
- - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the
- maximum acceptable input length for the model if that argument is not provided.
- - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of
- different lengths).
+ - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
+ sequence if provided).
+ - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
+ acceptable input length for the model if that argument is not provided.
+ - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different
+ lengths).
truncation (`bool`, `str` or [`TapasTruncationStrategy`], *optional*, defaults to `False`):
Activates and controls truncation. Accepts the following values:
- - `True` or `'drop_rows_to_fit'`: Truncate to a maximum length specified with the argument
- `max_length` or to the maximum acceptable input length for the model if that argument is not
- provided. This will truncate row by row, removing rows from the table.
- - `False` or `'do_not_truncate'` (default): No truncation (i.e., can output batch with
- sequence lengths greater than the model maximum admissible input size).
+ - `True` or `'drop_rows_to_fit'`: Truncate to a maximum length specified with the argument `max_length`
+ or to the maximum acceptable input length for the model if that argument is not provided. This will
+ truncate row by row, removing rows from the table.
+ - `False` or `'do_not_truncate'` (default): No truncation (i.e., can output batch with sequence lengths
+ greater than the model maximum admissible input size).
max_length (`int`, *optional*):
Controls the maximum length to use by one of the truncation/padding parameters.
- If left unset or set to `None`, this will use the predefined model maximum length if a maximum
- length is required by one of the truncation/padding parameters. If the model has no specific maximum
- input length (like XLNet) truncation/padding to a maximum length will be deactivated.
+ If left unset or set to `None`, this will use the predefined model maximum length if a maximum length
+ is required by one of the truncation/padding parameters. If the model has no specific maximum input
+ length (like XLNet) truncation/padding to a maximum length will be deactivated.
is_split_into_words (`bool`, *optional*, defaults to `False`):
Whether or not the input is already pre-tokenized (e.g., split into words). If set to `True`, the
tokenizer assumes the input is already split into words (for instance, by splitting it on whitespace)
@@ -192,11 +191,10 @@ class TapasTokenizer(PreTrainedTokenizer):
Construct a TAPAS tokenizer. Based on WordPiece. Flattens a table and one or more related sentences to be used by
TAPAS models.
- This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods.
- Users should refer to this superclass for more information regarding those methods.
- [`TapasTokenizer`] creates several token type ids to encode tabular structure. To be more
- precise, it adds 7 token type ids, in the following order: `segment_ids`, `column_ids`, `row_ids`,
- `prev_labels`, `column_ranks`, `inv_column_ranks` and `numeric_relations`:
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
+ this superclass for more information regarding those methods. [`TapasTokenizer`] creates several token type ids to
+ encode tabular structure. To be more precise, it adds 7 token type ids, in the following order: `segment_ids`,
+ `column_ids`, `row_ids`, `prev_labels`, `column_ranks`, `inv_column_ranks` and `numeric_relations`:
- segment_ids: indicate whether a token belongs to the question (0) or the table (1). 0 for special tokens and
padding.
@@ -215,8 +213,8 @@ class TapasTokenizer(PreTrainedTokenizer):
- numeric_relations: indicate numeric relations between the question and the tokens of the table. 0 for all
question tokens, special tokens and padding.
- [`TapasTokenizer`] runs end-to-end tokenization on a table and associated sentences: punctuation
- splitting and wordpiece.
+ [`TapasTokenizer`] runs end-to-end tokenization on a table and associated sentences: punctuation splitting and
+ wordpiece.
Args:
vocab_file (`str`):
@@ -1285,8 +1283,8 @@ class TapasTokenizer(PreTrainedTokenizer):
available strategy is the `"drop_rows_to_fit"` strategy.
Returns:
- `Tuple(int, int)`: tuple containing the number of rows after truncation, and the number of tokens
- available for each table element.
+ `Tuple(int, int)`: tuple containing the number of rows after truncation, and the number of tokens available
+ for each table element.
"""
if not isinstance(truncation_strategy, TapasTruncationStrategy):
truncation_strategy = TapasTruncationStrategy(truncation_strategy)
@@ -1370,8 +1368,8 @@ class TapasTokenizer(PreTrainedTokenizer):
Args:
question_tokens (`List[String]`):
- List of question tokens. Returns: `int`: the number of tokens left for the table, given the model
- max length.
+ List of question tokens. Returns: `int`: the number of tokens left for the table, given the model max
+ length.
"""
return (max_length if max_length is not None else self.model_max_length) - self._question_encoding_cost(
question_tokens
@@ -1791,7 +1789,8 @@ class TapasTokenizer(PreTrainedTokenizer):
Pad encoded inputs (on left/right and up to predefined length or max length in the batch)
Args:
- encoded_inputs: Dictionary of tokenized inputs (`List[int]`) or batch of tokenized inputs (`List[List[int]]`).
+ encoded_inputs:
+ Dictionary of tokenized inputs (`List[int]`) or batch of tokenized inputs (`List[List[int]]`).
max_length: maximum length of the returned list and optionally padding length (see below).
Will truncate by taking into account the special tokens.
padding_strategy: PaddingStrategy to use for padding.
@@ -1806,7 +1805,8 @@ class TapasTokenizer(PreTrainedTokenizer):
pad_to_multiple_of: (optional) Integer if set will pad the sequence to a multiple of the provided value.
This is especially useful to enable the use of Tensor Core on NVIDIA hardware with compute capability
>= 7.5 (Volta).
- return_attention_mask: (optional) Set to False to avoid returning attention mask (default: set to model specifics)
+ return_attention_mask:
+ (optional) Set to False to avoid returning attention mask (default: set to model specifics)
"""
# Load from model defaults
if return_attention_mask is None:
@@ -1890,15 +1890,15 @@ class TapasTokenizer(PreTrainedTokenizer):
def convert_logits_to_predictions(self, data, logits, logits_agg=None, cell_classification_threshold=0.5):
"""
- Converts logits of [`TapasForQuestionAnswering`] to actual predicted answer coordinates and
- optional aggregation indices.
+ Converts logits of [`TapasForQuestionAnswering`] to actual predicted answer coordinates and optional
+ aggregation indices.
- The original implementation, on which this function is based, can be found [here](https://github.com/google-research/tapas/blob/4908213eb4df7aa988573350278b44c4dbe3f71b/tapas/experiments/prediction_utils.py#L288).
+ The original implementation, on which this function is based, can be found
+ [here](https://github.com/google-research/tapas/blob/4908213eb4df7aa988573350278b44c4dbe3f71b/tapas/experiments/prediction_utils.py#L288).
Args:
data (`dict`):
- Dictionary mapping features to actual values. Should be created using
- [`TapasTokenizer`].
+ Dictionary mapping features to actual values. Should be created using [`TapasTokenizer`].
logits (`torch.Tensor` or `tf.Tensor` of shape `(batch_size, sequence_length)`):
Tensor containing the logits at the token level.
logits_agg (`torch.Tensor` or `tf.Tensor` of shape `(batch_size, num_aggregation_labels)`, *optional*):
@@ -1910,10 +1910,9 @@ class TapasTokenizer(PreTrainedTokenizer):
Returns:
`tuple` comprising various elements depending on the inputs:
- - predicted_answer_coordinates (`List[List[[tuple]]` of length `batch_size`): Predicted answer
- coordinates as a list of lists of tuples. Each element in the list contains the predicted answer
- coordinates of a single example in the batch, as a list of tuples. Each tuple is a cell, i.e. (row index,
- column index).
+ - predicted_answer_coordinates (`List[List[[tuple]]` of length `batch_size`): Predicted answer coordinates
+ as a list of lists of tuples. Each element in the list contains the predicted answer coordinates of a
+ single example in the batch, as a list of tuples. Each tuple is a cell, i.e. (row index, column index).
- predicted_aggregation_indices (`List[int]`of length `batch_size`, *optional*, returned when
`logits_aggregation` is provided): Predicted aggregation operator indices of the aggregation head.
"""
@@ -2004,7 +2003,8 @@ class BasicTokenizer(object):
tokenize_chinese_chars (`bool`, *optional*, defaults to `True`):
Whether or not to tokenize Chinese characters.
- This should likely be deactivated for Japanese (see this [issue](https://github.com/huggingface/transformers/issues/328)).
+ This should likely be deactivated for Japanese (see this
+ [issue](https://github.com/huggingface/transformers/issues/328)).
strip_accents: (`bool`, *optional*):
Whether or not to strip all accents. If this option is not specified, then it will be determined by the
value for `lowercase` (as in the original BERT).
diff --git a/src/transformers/models/transfo_xl/configuration_transfo_xl.py b/src/transformers/models/transfo_xl/configuration_transfo_xl.py
index 47f824c6ab..de7a154469 100644
--- a/src/transformers/models/transfo_xl/configuration_transfo_xl.py
+++ b/src/transformers/models/transfo_xl/configuration_transfo_xl.py
@@ -13,7 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" Transformer XL configuration """
+""" Transformer XL configuration"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging
@@ -28,19 +28,18 @@ TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class TransfoXLConfig(PretrainedConfig):
"""
- This is the configuration class to store the configuration of a [`TransfoXLModel`] or a
- [`TFTransfoXLModel`]. It is used to instantiate a Transformer-XL model according to the
- specified arguments, defining the model architecture. Instantiating a configuration with the defaults will yield a
- similar configuration to that of the [Transformer XL](https://huggingface.co/transfo-xl-wt103) architecture.
+ This is the configuration class to store the configuration of a [`TransfoXLModel`] or a [`TFTransfoXLModel`]. It is
+ used to instantiate a Transformer-XL model according to the specified arguments, defining the model architecture.
+ Instantiating a configuration with the defaults will yield a similar configuration to that of the [Transformer
+ XL](https://huggingface.co/transfo-xl-wt103) architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
vocab_size (`int`, *optional*, defaults to 267735):
Vocabulary size of the BERT model. Defines the number of different tokens that can be represented by the
- `inputs_ids` passed when calling [`TransfoXLModel`] or
- [`TFTransfoXLModel`].
+ `inputs_ids` passed when calling [`TransfoXLModel`] or [`TFTransfoXLModel`].
cutoffs (`List[int]`, *optional*, defaults to `[20000, 40000, 200000]`):
Cutoffs for the adaptive softmax.
d_model (`int`, *optional*, defaults to 1024):
diff --git a/src/transformers/models/transfo_xl/modeling_tf_transfo_xl.py b/src/transformers/models/transfo_xl/modeling_tf_transfo_xl.py
index f9f7fd5af6..bffce06ff3 100644
--- a/src/transformers/models/transfo_xl/modeling_tf_transfo_xl.py
+++ b/src/transformers/models/transfo_xl/modeling_tf_transfo_xl.py
@@ -711,12 +711,13 @@ class TFTransfoXLModelOutput(ModelOutput):
input) to speed up sequential decoding. The token ids which have their past given to this model should not
be passed as input ids as they have already been computed.
hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -734,7 +735,7 @@ class TFTransfoXLLMHeadModelOutput(ModelOutput):
Base class for model's outputs that may also contain a past key/values (to speed up sequential decoding).
Args:
- losses (`tf.Tensor` of shape *(batch_size, sequence_length-1)*, *optional*, returned when `labels` is provided)
+ losses (`tf.Tensor` of shape *(batch_size, sequence_length-1)*, *optional*, returned when `labels` is provided):
Language modeling losses (not reduced).
prediction_scores (`tf.Tensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
Prediction scores of the language modeling head (scores for each vocabulary token after SoftMax).
@@ -743,12 +744,13 @@ class TFTransfoXLLMHeadModelOutput(ModelOutput):
input) to speed up sequential decoding. The token ids which have their past given to this model should not
be passed as input ids as they have already been computed.
hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -775,12 +777,13 @@ class TFTransfoXLSequenceClassifierOutputWithPast(ModelOutput):
input) to speed up sequential decoding. The token ids which have their past given to this model should not
be passed as input ids as they have already been computed.
hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -795,13 +798,13 @@ class TFTransfoXLSequenceClassifierOutputWithPast(ModelOutput):
TRANSFO_XL_START_DOCSTRING = r"""
- This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading or saving, resizing the input
- embeddings, pruning heads etc.)
+ This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use
- it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage
- and behavior.
+ This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it
+ as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and
+ behavior.
@@ -810,11 +813,11 @@ TRANSFO_XL_START_DOCSTRING = r"""
- having all inputs as keyword arguments (like PyTorch models), or
- having all inputs as a list, tuple or dict in the first positional arguments.
- This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all
- the tensors in the first argument of the model call function: `model(inputs)`.
+ This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all the
+ tensors in the first argument of the model call function: `model(inputs)`.
- If you choose this second option, there are three possibilities you can use to gather all the input Tensors in
- the first positional argument :
+ If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the
+ first positional argument :
- a single Tensor with `input_ids` only and nothing else: `model(inputs_ids)`
- a list of varying length with one or several input Tensors IN THE ORDER given in the docstring:
@@ -827,8 +830,7 @@ TRANSFO_XL_START_DOCSTRING = r"""
Parameters:
config ([`TransfoXLConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
TRANSFO_XL_INPUTS_DOCSTRING = r"""
@@ -836,9 +838,8 @@ TRANSFO_XL_INPUTS_DOCSTRING = r"""
input_ids (`tf.Tensor` or `Numpy array` of shape `(batch_size, sequence_length)`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`BertTokenizer`]. See
- [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for
- details.
+ Indices can be obtained using [`BertTokenizer`]. See [`PreTrainedTokenizer.__call__`] and
+ [`PreTrainedTokenizer.encode`] for details.
[What are input IDs?](../glossary#input-ids)
mems (`List[tf.Tensor]` of length `config.n_layers`):
@@ -851,9 +852,9 @@ TRANSFO_XL_INPUTS_DOCSTRING = r"""
- 1 indicates the head is **not masked**,
- 0 indicates the head is **masked**.
inputs_embeds (`tf.Tensor` or `Numpy array` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the
@@ -863,8 +864,8 @@ TRANSFO_XL_INPUTS_DOCSTRING = r"""
more detail. This argument can be used only in eager mode, in graph mode the value in the config will be
used instead.
return_dict (`bool`, *optional*):
- Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This
- argument can be used in eager mode, in graph mode the value will always be set to True.
+ Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This argument can be used
+ in eager mode, in graph mode the value will always be set to True.
training (`bool`, *optional*, defaults to `False`):
Whether or not to use the model in training mode (some modules like dropout modules have different
behaviors between training and evaluation).
@@ -1062,14 +1063,14 @@ class TFTransfoXLLMHeadModel(TFTransfoXLPreTrainedModel):
"""
The Transfo XL Model transformer with a sequence classification head on top (linear layer).
- [`TFTransfoXLForSequenceClassification`] uses the last token in order to do the classification,
- as other causal models (e.g. GPT-1,GPT-2) do.
+ [`TFTransfoXLForSequenceClassification`] uses the last token in order to do the classification, as other causal
+ models (e.g. GPT-1,GPT-2) do.
Since it does classification on the last token, it requires to know the position of the last token. If a
- `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each
- row. If no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot
- guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take
- the last value in each row of the batch).
+ `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
+ no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
+ padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
+ each row of the batch).
""",
TRANSFO_XL_START_DOCSTRING,
)
@@ -1110,7 +1111,8 @@ class TFTransfoXLForSequenceClassification(TFTransfoXLPreTrainedModel, TFSequenc
):
r"""
labels (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the cross entropy classification loss. Indices should be in `[0, ..., config.vocab_size - 1]`.
+ Labels for computing the cross entropy classification loss. Indices should be in `[0, ...,
+ config.vocab_size - 1]`.
"""
inputs = input_processing(
func=self.call,
diff --git a/src/transformers/models/transfo_xl/modeling_transfo_xl.py b/src/transformers/models/transfo_xl/modeling_transfo_xl.py
index 7a5eb7e6f3..25823dafac 100644
--- a/src/transformers/models/transfo_xl/modeling_transfo_xl.py
+++ b/src/transformers/models/transfo_xl/modeling_transfo_xl.py
@@ -615,12 +615,13 @@ class TransfoXLModelOutput(ModelOutput):
input) to speed up sequential decoding. The token ids which have their past given to this model should not
be passed as input ids as they have already been computed.
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -647,12 +648,13 @@ class TransfoXLSequenceClassifierOutputWithPast(ModelOutput):
input) to speed up sequential decoding. The token ids which have their past given to this model should not
be passed as input ids as they have already been computed.
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -671,7 +673,7 @@ class TransfoXLLMHeadModelOutput(ModelOutput):
Base class for model's outputs that may also contain a past key/values (to speed up sequential decoding).
Args:
- losses (`torch.FloatTensor` of shape *(batch_size, sequence_length-1)*, *optional*, returned when `labels` is provided)
+ losses (`torch.FloatTensor` of shape *(batch_size, sequence_length-1)*, *optional*, returned when `labels` is provided):
Language modeling losses (not reduced).
prediction_scores (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
Prediction scores of the language modeling head (scores for each vocabulary token after SoftMax).
@@ -680,12 +682,13 @@ class TransfoXLLMHeadModelOutput(ModelOutput):
input) to speed up sequential decoding. The token ids which have their past given to this model should not
be passed as input ids as they have already been computed.
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -709,19 +712,18 @@ class TransfoXLLMHeadModelOutput(ModelOutput):
TRANSFO_XL_START_DOCSTRING = r"""
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
Parameters:
config ([`TransfoXLConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
TRANSFO_XL_INPUTS_DOCSTRING = r"""
@@ -729,9 +731,8 @@ TRANSFO_XL_INPUTS_DOCSTRING = r"""
input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`TransfoXLTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`TransfoXLTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
mems (`List[torch.FloatTensor]` of length `config.n_layers`):
@@ -745,9 +746,9 @@ TRANSFO_XL_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -1070,8 +1071,8 @@ class TransfoXLLMHeadModel(TransfoXLPreTrainedModel):
r"""
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
- `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to
- `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
+ `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
+ are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
if input_ids is not None:
@@ -1140,8 +1141,8 @@ class TransfoXLLMHeadModel(TransfoXLPreTrainedModel):
def _reorder_cache(mems: List[torch.Tensor], beam_idx: torch.Tensor) -> List[torch.Tensor]:
"""
This function is used to re-order the `mems` cache if [`~PreTrainedModel.beam_search`] or
- [`~PreTrainedModel.beam_sample`] is called. This is required to match `mems` with the
- correct beam_idx at every generation step.
+ [`~PreTrainedModel.beam_sample`] is called. This is required to match `mems` with the correct beam_idx at every
+ generation step.
"""
return [layer_past.index_select(1, beam_idx.to(layer_past.device)) for layer_past in mems]
@@ -1150,14 +1151,14 @@ class TransfoXLLMHeadModel(TransfoXLPreTrainedModel):
"""
The Transformer-XL Model transformer with a sequence classification head on top (linear layer).
- [`TransfoXLForSequenceClassification`] uses the last token in order to do the classification, as
- other causal models (e.g. GPT-1) do.
+ [`TransfoXLForSequenceClassification`] uses the last token in order to do the classification, as other causal
+ models (e.g. GPT-1) do.
Since it does classification on the last token, it requires to know the position of the last token. If a
- `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each
- row. If no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot
- guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take
- the last value in each row of the batch).
+ `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
+ no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
+ padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
+ each row of the batch).
""",
TRANSFO_XL_START_DOCSTRING,
)
@@ -1192,8 +1193,9 @@ class TransfoXLForSequenceClassification(TransfoXLPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
diff --git a/src/transformers/models/transfo_xl/modeling_transfo_xl_utilities.py b/src/transformers/models/transfo_xl/modeling_transfo_xl_utilities.py
index ca7a1b5621..dad3183a9b 100644
--- a/src/transformers/models/transfo_xl/modeling_transfo_xl_utilities.py
+++ b/src/transformers/models/transfo_xl/modeling_transfo_xl_utilities.py
@@ -196,8 +196,8 @@ class ProjectedAdaptiveLogSoftmax(nn.Module):
hidden (Tensor): a minibatch of example
Returns:
- log-probabilities of for each class \\(c\\) in range \\(0 <= c <= n\_classes\\), where
- \\(n\_classes\\) is a parameter passed to `AdaptiveLogSoftmaxWithLoss` constructor. Shape:
+ log-probabilities of for each class \\(c\\) in range \\(0 <= c <= n\_classes\\), where \\(n\_classes\\) is
+ a parameter passed to `AdaptiveLogSoftmaxWithLoss` constructor. Shape:
- Input: \\((N, in\_features)\\)
- Output: \\((N, n\_classes)\\)
diff --git a/src/transformers/models/transfo_xl/tokenization_transfo_xl.py b/src/transformers/models/transfo_xl/tokenization_transfo_xl.py
index f5f1412148..5eb0c1b41c 100644
--- a/src/transformers/models/transfo_xl/tokenization_transfo_xl.py
+++ b/src/transformers/models/transfo_xl/tokenization_transfo_xl.py
@@ -114,11 +114,12 @@ def detokenize_numbers(text: str) -> str:
class TransfoXLTokenizer(PreTrainedTokenizer):
"""
- Construct a Transformer-XL tokenizer adapted from Vocab class in [the original code](https://github.com/kimiyoung/transformer-xl). The Transformer-XL tokenizer is a word-level tokenizer (no
+ Construct a Transformer-XL tokenizer adapted from Vocab class in [the original
+ code](https://github.com/kimiyoung/transformer-xl). The Transformer-XL tokenizer is a word-level tokenizer (no
sub-word tokenization).
- This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods.
- Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
+ this superclass for more information regarding those methods.
Args:
special (`List[str]`, *optional*):
@@ -410,10 +411,10 @@ class TransfoXLTokenizer(PreTrainedTokenizer):
def moses_pipeline(self, text: str) -> List[str]:
"""
- Does basic tokenization using [`sacremoses.MosesPunctNormalizer`] and [`sacremoses.MosesTokenizer`]
- with *aggressive_dash_splits=True* (see [`sacremoses.tokenize.MosesTokenizer.tokenize`]). Additionally,
- large comma-separated numbers and floating point values are split. E.g. "23,000 people are 1.80m tall" -> "23
- @,@ 000 people are 1 @.@ 80m tall"
+ Does basic tokenization using [`sacremoses.MosesPunctNormalizer`] and [`sacremoses.MosesTokenizer`] with
+ *aggressive_dash_splits=True* (see [`sacremoses.tokenize.MosesTokenizer.tokenize`]). Additionally, large
+ comma-separated numbers and floating point values are split. E.g. "23,000 people are 1.80m tall" -> "23 @,@ 000
+ people are 1 @.@ 80m tall"
Args:
text: Text to be tokenize
diff --git a/src/transformers/models/trocr/configuration_trocr.py b/src/transformers/models/trocr/configuration_trocr.py
index 6b6c2d180a..f8474489f0 100644
--- a/src/transformers/models/trocr/configuration_trocr.py
+++ b/src/transformers/models/trocr/configuration_trocr.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" TrOCR model configuration """
+""" TrOCR model configuration"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging
@@ -28,12 +28,13 @@ TROCR_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class TrOCRConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`TrOCRForCausalLM`]. It is used
- to instantiate an TrOCR model according to the specified arguments, defining the model architecture. Instantiating
- a configuration with the defaults will yield a similar configuration to that of the TrOCR [microsoft/trocr-base](https://huggingface.co/microsoft/trocr-base) architecture.
+ This is the configuration class to store the configuration of a [`TrOCRForCausalLM`]. It is used to instantiate an
+ TrOCR model according to the specified arguments, defining the model architecture. Instantiating a configuration
+ with the defaults will yield a similar configuration to that of the TrOCR
+ [microsoft/trocr-base](https://huggingface.co/microsoft/trocr-base) architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
@@ -49,8 +50,8 @@ class TrOCRConfig(PretrainedConfig):
decoder_ffn_dim (`int`, *optional*, defaults to 4096):
Dimensionality of the "intermediate" (often named feed-forward) layer in decoder.
activation_function (`str` or `function`, *optional*, defaults to `"gelu"`):
- The non-linear activation function (function or string) in the pooler. If string, `"gelu"`,
- `"relu"`, `"silu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the pooler. If string, `"gelu"`, `"relu"`,
+ `"silu"` and `"gelu_new"` are supported.
max_position_embeddings (`int`, *optional*, defaults to 512):
The maximum sequence length that this model might ever be used with. Typically set this to something large
just in case (e.g., 512 or 1024 or 2048).
@@ -65,8 +66,8 @@ class TrOCRConfig(PretrainedConfig):
init_std (`float`, *optional*, defaults to 0.02):
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
decoder_layerdrop: (`float`, *optional*, defaults to 0.0):
- The LayerDrop probability for the decoder. See the [LayerDrop paper](see
- https://arxiv.org/abs/1909.11556) for more details.
+ The LayerDrop probability for the decoder. See the [LayerDrop paper](see https://arxiv.org/abs/1909.11556)
+ for more details.
use_cache (`bool`, *optional*, defaults to `True`):
Whether or not the model should return the last key/values attentions (not used by all models).
scale_embedding (`bool`, *optional*, defaults to `False`):
diff --git a/src/transformers/models/trocr/modeling_trocr.py b/src/transformers/models/trocr/modeling_trocr.py
index 32eccc9c87..c1a82aaf5c 100644
--- a/src/transformers/models/trocr/modeling_trocr.py
+++ b/src/transformers/models/trocr/modeling_trocr.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" PyTorch TrOCR decoder model (based on RoBERTa). """
+""" PyTorch TrOCR decoder model (based on RoBERTa)."""
import copy
@@ -356,7 +356,8 @@ class TrOCRDecoderLayer(nn.Module):
hidden_states (`torch.FloatTensor`): input to the layer of shape `(seq_len, batch, embed_dim)`
attention_mask (`torch.FloatTensor`): attention mask of size
`(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
- encoder_hidden_states (`torch.FloatTensor`): cross attention input to the layer of shape `(seq_len, batch, embed_dim)`
+ encoder_hidden_states (`torch.FloatTensor`):
+ cross attention input to the layer of shape `(seq_len, batch, embed_dim)`
encoder_attention_mask (`torch.FloatTensor`): encoder attention mask of size
`(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
layer_head_mask (`torch.FloatTensor`): mask for attention heads in a given layer of size
@@ -454,13 +455,13 @@ class TrOCRPreTrainedModel(PreTrainedModel):
TROCR_START_DOCSTRING = r"""
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
Parameters:
config ([`TrOCRConfig`]):
@@ -552,9 +553,8 @@ class TrOCRDecoder(TrOCRPreTrainedModel):
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
provide it.
- Indices can be obtained using [`TrOCRTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`]
- for details.
+ Indices can be obtained using [`TrOCRTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -589,19 +589,20 @@ class TrOCRDecoder(TrOCRPreTrainedModel):
- 0 indicates the head is **masked**.
past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2
- tensors of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional
- tensors of shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
+ Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of
+ shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
+ shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
Contains pre-computed hidden-states (key and values in the self-attention blocks and in the
- cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential
- decoding.
+ cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
- If `past_key_values` are used, the user can optionally input only the last
- `decoder_input_ids` (those that don't have their past key value states given to this model) of
- shape `(batch_size, 1)` instead of all ``decoder_input_ids``` of shape `(batch_size,
- sequence_length)`. inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices
- into associated vectors than the model's internal embedding lookup matrix.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those
+ that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of
+ all ``decoder_input_ids``` of shape `(batch_size, sequence_length)`. inputs_embeds (`torch.FloatTensor`
+ of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing
+ `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more
+ control over how to convert `input_ids` indices into associated vectors than the model's internal
+ embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under
returned tensors for more detail.
@@ -826,9 +827,8 @@ class TrOCRForCausalLM(TrOCRPreTrainedModel):
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
provide it.
- Indices can be obtained using [`TrOCRTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`]
- for details.
+ Indices can be obtained using [`TrOCRTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -857,25 +857,24 @@ class TrOCRForCausalLM(TrOCRPreTrainedModel):
- 0 indicates the head is **masked**.
past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
- Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2
- tensors of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional
- tensors of shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. The two
- additional tensors are only required when the model is used as a decoder in a Sequence to Sequence
- model.
+ Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of
+ shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of
+ shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. The two additional
+ tensors are only required when the model is used as a decoder in a Sequence to Sequence model.
Contains pre-computed hidden-states (key and values in the self-attention blocks and in the
- cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential
- decoding.
+ cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those
+ that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of
+ all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are
- ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
+ Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
+ config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
+ (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
use_cache (`bool`, *optional*):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`).
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
+ (see `past_key_values`).
- 1 for tokens that are **not masked**,
- 0 for tokens that are **masked**.
diff --git a/src/transformers/models/trocr/processing_trocr.py b/src/transformers/models/trocr/processing_trocr.py
index 24f8fc0dd7..3166cbae20 100644
--- a/src/transformers/models/trocr/processing_trocr.py
+++ b/src/transformers/models/trocr/processing_trocr.py
@@ -28,9 +28,8 @@ class TrOCRProcessor:
r"""
Constructs a TrOCR processor which wraps a vision feature extractor and a TrOCR tokenizer into a single processor.
- [`TrOCRProcessor`] offers all the functionalities of [`AutoFeatureExtractor`]
- and [`RobertaTokenizer`]. See the [`~TrOCRProcessor.__call__`] and
- [`~TrOCRProcessor.decode`] for more information.
+ [`TrOCRProcessor`] offers all the functionalities of [`AutoFeatureExtractor`] and [`RobertaTokenizer`]. See the
+ [`~TrOCRProcessor.__call__`] and [`~TrOCRProcessor.decode`] for more information.
Args:
feature_extractor ([`AutoFeatureExtractor`]):
@@ -55,14 +54,14 @@ class TrOCRProcessor:
def save_pretrained(self, save_directory):
"""
- Save a TrOCR feature extractor object and TrOCR tokenizer object to the directory `save_directory`, so that
- it can be re-loaded using the [`~TrOCRProcessor.from_pretrained`] class method.
+ Save a TrOCR feature extractor object and TrOCR tokenizer object to the directory `save_directory`, so that it
+ can be re-loaded using the [`~TrOCRProcessor.from_pretrained`] class method.
This class method is simply calling [`~PreTrainedFeatureExtractor.save_pretrained`] and
- [`~tokenization_utils_base.PreTrainedTokenizer.save_pretrained`]. Please refer to the
- docstrings of the methods above for more information.
+ [`~tokenization_utils_base.PreTrainedTokenizer.save_pretrained`]. Please refer to the docstrings of the methods
+ above for more information.
@@ -82,9 +81,8 @@ class TrOCRProcessor:
- This class method is simply calling AutoFeatureExtractor's
- [`~PreTrainedFeatureExtractor.from_pretrained`] and TrOCRTokenizer's
- [`~tokenization_utils_base.PreTrainedTokenizer.from_pretrained`]. Please refer to the
+ This class method is simply calling AutoFeatureExtractor's [`~PreTrainedFeatureExtractor.from_pretrained`] and
+ TrOCRTokenizer's [`~tokenization_utils_base.PreTrainedTokenizer.from_pretrained`]. Please refer to the
docstrings of the methods above for more information.
@@ -97,8 +95,7 @@ class TrOCRProcessor:
huggingface.co. Valid model ids can be located at the root-level, like `bert-base-uncased`, or
namespaced under a user or organization name, like `dbmdz/bert-base-german-cased`.
- a path to a *directory* containing a feature extractor file saved using the
- [`~PreTrainedFeatureExtractor.save_pretrained`] method, e.g.,
- `./my_model_directory/`.
+ [`~PreTrainedFeatureExtractor.save_pretrained`] method, e.g., `./my_model_directory/`.
- a path or url to a saved feature extractor JSON *file*, e.g.,
`./my_model_directory/preprocessor_config.json`.
**kwargs
@@ -114,24 +111,22 @@ class TrOCRProcessor:
"""
When used in normal mode, this method forwards all its arguments to AutoFeatureExtractor's
[`~AutoFeatureExtractor.__call__`] and returns its output. If used in the context
- [`~TrOCRProcessor.as_target_processor`] this method forwards all its arguments to
- TrOCRTokenizer's [`~TrOCRTokenizer.__call__`]. Please refer to the doctsring of the above two
- methods for more information.
+ [`~TrOCRProcessor.as_target_processor`] this method forwards all its arguments to TrOCRTokenizer's
+ [`~TrOCRTokenizer.__call__`]. Please refer to the doctsring of the above two methods for more information.
"""
return self.current_processor(*args, **kwargs)
def batch_decode(self, *args, **kwargs):
"""
- This method forwards all its arguments to TrOCRTokenizer's
- [`~PreTrainedTokenizer.batch_decode`]. Please refer to the docstring of this method for more
- information.
+ This method forwards all its arguments to TrOCRTokenizer's [`~PreTrainedTokenizer.batch_decode`]. Please refer
+ to the docstring of this method for more information.
"""
return self.tokenizer.batch_decode(*args, **kwargs)
def decode(self, *args, **kwargs):
"""
- This method forwards all its arguments to TrOCRTokenizer's [`~PreTrainedTokenizer.decode`].
- Please refer to the docstring of this method for more information.
+ This method forwards all its arguments to TrOCRTokenizer's [`~PreTrainedTokenizer.decode`]. Please refer to the
+ docstring of this method for more information.
"""
return self.tokenizer.decode(*args, **kwargs)
diff --git a/src/transformers/models/unispeech/configuration_unispeech.py b/src/transformers/models/unispeech/configuration_unispeech.py
index 0d1a08d653..5579512c2b 100644
--- a/src/transformers/models/unispeech/configuration_unispeech.py
+++ b/src/transformers/models/unispeech/configuration_unispeech.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" UniSpeech model configuration """
+""" UniSpeech model configuration"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging
@@ -28,21 +28,21 @@ UNISPEECH_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class UniSpeechConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`UniSpeechModel`]. It is used
- to instantiate an UniSpeech model according to the specified arguments, defining the model architecture.
- Instantiating a configuration with the defaults will yield a similar configuration to that of the UniSpeech
+ This is the configuration class to store the configuration of a [`UniSpeechModel`]. It is used to instantiate an
+ UniSpeech model according to the specified arguments, defining the model architecture. Instantiating a
+ configuration with the defaults will yield a similar configuration to that of the UniSpeech
[facebook/unispeech-base-960h](https://huggingface.co/facebook/unispeech-base-960h) architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
vocab_size (`int`, *optional*, defaults to 32):
Vocabulary size of the UniSpeech model. Defines the number of different tokens that can be represented by
- the `inputs_ids` passed when calling [`UniSpeechModel`]. Vocabulary size of the
- model. Defines the different tokens that can be represented by the *inputs_ids* passed to the forward
- method of [`UniSpeechModel`].
+ the `inputs_ids` passed when calling [`UniSpeechModel`]. Vocabulary size of the model. Defines the
+ different tokens that can be represented by the *inputs_ids* passed to the forward method of
+ [`UniSpeechModel`].
hidden_size (`int`, *optional*, defaults to 768):
Dimensionality of the encoder layers and the pooler layer.
num_hidden_layers (`int`, *optional*, defaults to 12):
@@ -52,8 +52,8 @@ class UniSpeechConfig(PretrainedConfig):
intermediate_size (`int`, *optional*, defaults to 3072):
Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"selu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"selu"` and `"gelu_new"` are supported.
hidden_dropout (`float`, *optional*, defaults to 0.1):
The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
attention_dropout (`float`, *optional*, defaults to 0.1):
@@ -93,28 +93,32 @@ class UniSpeechConfig(PretrainedConfig):
num_conv_pos_embedding_groups (`int`, *optional*, defaults to 16):
Number of groups of 1D convolutional positional embeddings layer.
do_stable_layer_norm (`bool`, *optional*, defaults to `False`):
- Whether to apply *stable* layer norm architecture of the Transformer encoder. `do_stable_layer_norm is True` corresponds to applying layer norm before the attention layer, whereas `do_stable_layer_norm is False` corresponds to applying layer norm after the attention layer.
+ Whether to apply *stable* layer norm architecture of the Transformer encoder. `do_stable_layer_norm is
+ True` corresponds to applying layer norm before the attention layer, whereas `do_stable_layer_norm is
+ False` corresponds to applying layer norm after the attention layer.
apply_spec_augment (`bool`, *optional*, defaults to `True`):
Whether to apply *SpecAugment* data augmentation to the outputs of the feature extractor. For reference see
- [SpecAugment: A Simple Data Augmentation Method for Automatic Speech Recognition](https://arxiv.org/abs/1904.08779).
+ [SpecAugment: A Simple Data Augmentation Method for Automatic Speech
+ Recognition](https://arxiv.org/abs/1904.08779).
mask_time_prob (`float`, *optional*, defaults to 0.05):
Percentage (between 0 and 1) of all feature vectors along the time axis which will be masked. The masking
procecure generates ''mask_time_prob*len(time_axis)/mask_time_length'' independent masks over the axis. If
reasoning from the propability of each feature vector to be chosen as the start of the vector span to be
- masked, *mask_time_prob* should be `prob_vector_start*mask_time_length`. Note that overlap may decrease
- the actual percentage of masked vectors. This is only relevant if `apply_spec_augment is True`.
+ masked, *mask_time_prob* should be `prob_vector_start*mask_time_length`. Note that overlap may decrease the
+ actual percentage of masked vectors. This is only relevant if `apply_spec_augment is True`.
mask_time_length (`int`, *optional*, defaults to 10):
Length of vector span along the time axis.
mask_time_min_masks (`int`, *optional*, defaults to 2),:
- The minimum number of masks of length `mask_feature_length` generated along the time axis, each time
- step, irrespectively of `mask_feature_prob`. Only relevant if
- ''mask_time_prob*len(time_axis)/mask_time_length < mask_time_min_masks''
+ The minimum number of masks of length `mask_feature_length` generated along the time axis, each time step,
+ irrespectively of `mask_feature_prob`. Only relevant if ''mask_time_prob*len(time_axis)/mask_time_length <
+ mask_time_min_masks''
mask_feature_prob (`float`, *optional*, defaults to 0.0):
Percentage (between 0 and 1) of all feature vectors along the feature axis which will be masked. The
masking procecure generates ''mask_feature_prob*len(feature_axis)/mask_time_length'' independent masks over
the axis. If reasoning from the propability of each feature vector to be chosen as the start of the vector
- span to be masked, *mask_feature_prob* should be `prob_vector_start*mask_feature_length`. Note that
- overlap may decrease the actual percentage of masked vectors. This is only relevant if `apply_spec_augment is True`.
+ span to be masked, *mask_feature_prob* should be `prob_vector_start*mask_feature_length`. Note that overlap
+ may decrease the actual percentage of masked vectors. This is only relevant if `apply_spec_augment is
+ True`.
mask_feature_length (`int`, *optional*, defaults to 10):
Length of vector span along the feature axis.
mask_feature_min_masks (`int`, *optional*, defaults to 0),:
@@ -141,9 +145,9 @@ class UniSpeechConfig(PretrainedConfig):
Specifies the reduction to apply to the output of `torch.nn.CTCLoss`. Only relevant when training an
instance of [`UniSpeechForCTC`].
ctc_zero_infinity (`bool`, *optional*, defaults to `False`):
- Whether to zero infinite losses and the associated gradients of `torch.nn.CTCLoss`. Infinite losses
- mainly occur when the inputs are too short to be aligned to the targets. Only relevant when training an
- instance of [`UniSpeechForCTC`].
+ Whether to zero infinite losses and the associated gradients of `torch.nn.CTCLoss`. Infinite losses mainly
+ occur when the inputs are too short to be aligned to the targets. Only relevant when training an instance
+ of [`UniSpeechForCTC`].
use_weighted_layer_sum (`bool`, *optional*, defaults to `False`):
Whether to use a weighted average of layer outputs with learned weights. Only relevant when using an
instance of [`UniSpeechForSequenceClassification`].
diff --git a/src/transformers/models/unispeech/modeling_unispeech.py b/src/transformers/models/unispeech/modeling_unispeech.py
index 0e112019ec..1dbd7718a7 100755
--- a/src/transformers/models/unispeech/modeling_unispeech.py
+++ b/src/transformers/models/unispeech/modeling_unispeech.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" PyTorch UniSpeech model. """
+""" PyTorch UniSpeech model."""
import math
from dataclasses import dataclass
@@ -68,12 +68,13 @@ class UniSpeechBaseModelOutput(ModelOutput):
extract_features (`torch.FloatTensor` of shape `(batch_size, sequence_length, conv_dim[-1])`):
Sequence of extracted feature vectors of the last convolutional layer of the model.
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -101,12 +102,13 @@ class UniSpeechForPreTrainingOutput(ModelOutput):
Quantized extracted feature vectors projected to *config.proj_codevector_dim* representing the positive
target vectors for contrastive loss.
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -130,8 +132,8 @@ def _compute_mask_indices(
) -> np.ndarray:
"""
Computes random mask spans for a given shape. Used to implement [SpecAugment: A Simple Data Augmentation Method for
- ASR](https://arxiv.org/abs/1904.08779). Note that this method is not optimized to run on TPU and should be run
- on CPU as part of the preprocessing during training.
+ ASR](https://arxiv.org/abs/1904.08779). Note that this method is not optimized to run on TPU and should be run on
+ CPU as part of the preprocessing during training.
Args:
shape: The shape for which to compute masks. This should be of a tuple of size 2 where
@@ -983,11 +985,12 @@ class UniSpeechPreTrainedModel(PreTrainedModel):
UNISPEECH_START_DOCSTRING = r"""
- UniSpeech was proposed in [UniSpeech: Unified Speech Representation Learning with Labeled and Unlabeled Data](https://arxiv.org/abs/2101.07597) by Chengyi Wang, Yu Wu, Yao Qian, Kenichi Kumatani, Shujie Liu, Furu Wei,
+ UniSpeech was proposed in [UniSpeech: Unified Speech Representation Learning with Labeled and Unlabeled
+ Data](https://arxiv.org/abs/2101.07597) by Chengyi Wang, Yu Wu, Yao Qian, Kenichi Kumatani, Shujie Liu, Furu Wei,
Michael Zeng, Xuedong Huang.
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving etc.).
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving etc.).
This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use
it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
@@ -996,8 +999,7 @@ UNISPEECH_START_DOCSTRING = r"""
Parameters:
config ([`UniSpeechConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
@@ -1006,11 +1008,12 @@ UNISPEECH_INPUTS_DOCSTRING = r"""
input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`):
Float values of input raw speech waveform. Values can be obtained by loading a *.flac* or *.wav* audio file
into an array of type *List[float]* or a *numpy.ndarray*, *e.g.* via the soundfile library (*pip install
- soundfile*). To prepare the array into *input_values*, the [`UniSpeechProcessor`] should
- be used for padding and conversion into a tensor of type *torch.FloatTensor*. See
- [`UniSpeechProcessor.__call__`] for details.
+ soundfile*). To prepare the array into *input_values*, the [`UniSpeechProcessor`] should be used for
+ padding and conversion into a tensor of type *torch.FloatTensor*. See [`UniSpeechProcessor.__call__`] for
+ details.
attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0, 1]`:
+ Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0,
+ 1]`:
- 1 for tokens that are **not masked**,
- 0 for tokens that are **masked**.
@@ -1019,12 +1022,11 @@ UNISPEECH_INPUTS_DOCSTRING = r"""
- `attention_mask` should only be passed if the corresponding processor has
- `config.return_attention_mask == True`. For all models whose processor has
- `config.return_attention_mask == False`, `attention_mask` should **not** be passed to avoid
- degraded performance when doing batched inference. For such models `input_values` should simply be
- padded with 0 and passed without `attention_mask`. Be aware that these models also yield slightly
- different results depending on whether `input_values` is padded or not.
+ `attention_mask` should only be passed if the corresponding processor has `config.return_attention_mask ==
+ True`. For all models whose processor has `config.return_attention_mask == False`, `attention_mask` should
+ **not** be passed to avoid degraded performance when doing batched inference. For such models
+ `input_values` should simply be padded with 0 and passed without `attention_mask`. Be aware that these
+ models also yield slightly different results depending on whether `input_values` is padded or not.
@@ -1164,7 +1166,7 @@ class UniSpeechModel(UniSpeechPreTrainedModel):
@add_start_docstrings(
- """UniSpeech Model with a vector-quantization module and ctc loss for pre-training. """, UNISPEECH_START_DOCSTRING
+ """UniSpeech Model with a vector-quantization module and ctc loss for pre-training.""", UNISPEECH_START_DOCSTRING
)
class UniSpeechForPreTraining(UniSpeechPreTrainedModel):
def __init__(self, config: UniSpeechConfig):
@@ -1204,8 +1206,7 @@ class UniSpeechForPreTraining(UniSpeechPreTrainedModel):
):
"""
Compute logits for contrastive loss based using cosine similarity as the distance measure between
- `[positive_feature, negative_features]` and `[predicted_features]`. Additionally, temperature can be
- applied.
+ `[positive_feature, negative_features]` and `[predicted_features]`. Additionally, temperature can be applied.
"""
target_features = torch.cat([target_features, negative_features], dim=0)
@@ -1333,7 +1334,7 @@ class UniSpeechForPreTraining(UniSpeechPreTrainedModel):
@add_start_docstrings(
- """UniSpeech Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC). """,
+ """UniSpeech Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC).""",
UNISPEECH_START_DOCSTRING,
)
# Copied from transformers.models.wav2vec2.modeling_wav2vec2.Wav2Vec2ForCTC with Wav2Vec2->UniSpeech, wav2vec2->unispeech, WAV_2_VEC_2->UNISPEECH
@@ -1382,7 +1383,9 @@ class UniSpeechForCTC(UniSpeechPreTrainedModel):
r"""
labels (`torch.LongTensor` of shape `(batch_size, target_length)`, *optional*):
Labels for connectionist temporal classification. Note that `target_length` has to be smaller or equal to
- the sequence length of the output logits. Indices are selected in `[-100, 0, ..., config.vocab_size - 1]`. All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size - 1]`.
+ the sequence length of the output logits. Indices are selected in `[-100, 0, ..., config.vocab_size - 1]`.
+ All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ...,
+ config.vocab_size - 1]`.
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1497,8 +1500,9 @@ class UniSpeechForSequenceClassification(UniSpeechPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
diff --git a/src/transformers/models/unispeech_sat/configuration_unispeech_sat.py b/src/transformers/models/unispeech_sat/configuration_unispeech_sat.py
index 120fe49547..9b177b7da0 100644
--- a/src/transformers/models/unispeech_sat/configuration_unispeech_sat.py
+++ b/src/transformers/models/unispeech_sat/configuration_unispeech_sat.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" UniSpeechSat model configuration """
+""" UniSpeechSat model configuration"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging
@@ -28,21 +28,21 @@ UNISPEECH_SAT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class UniSpeechSatConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`UniSpeechSatModel`]. It is
- used to instantiate an UniSpeechSat model according to the specified arguments, defining the model architecture.
- Instantiating a configuration with the defaults will yield a similar configuration to that of the UniSpeechSat
+ This is the configuration class to store the configuration of a [`UniSpeechSatModel`]. It is used to instantiate an
+ UniSpeechSat model according to the specified arguments, defining the model architecture. Instantiating a
+ configuration with the defaults will yield a similar configuration to that of the UniSpeechSat
[facebook/unispeech_sat-base-960h](https://huggingface.co/facebook/unispeech_sat-base-960h) architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
vocab_size (`int`, *optional*, defaults to 32):
Vocabulary size of the UniSpeechSat model. Defines the number of different tokens that can be represented
- by the `inputs_ids` passed when calling [`UniSpeechSatModel`]. Vocabulary size of
- the model. Defines the different tokens that can be represented by the *inputs_ids* passed to the forward
- method of [`UniSpeechSatModel`].
+ by the `inputs_ids` passed when calling [`UniSpeechSatModel`]. Vocabulary size of the model. Defines the
+ different tokens that can be represented by the *inputs_ids* passed to the forward method of
+ [`UniSpeechSatModel`].
hidden_size (`int`, *optional*, defaults to 768):
Dimensionality of the encoder layers and the pooler layer.
num_hidden_layers (`int`, *optional*, defaults to 12):
@@ -52,8 +52,8 @@ class UniSpeechSatConfig(PretrainedConfig):
intermediate_size (`int`, *optional*, defaults to 3072):
Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"selu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"selu"` and `"gelu_new"` are supported.
hidden_dropout (`float`, *optional*, defaults to 0.1):
The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
attention_dropout (`float`, *optional*, defaults to 0.1):
@@ -93,28 +93,32 @@ class UniSpeechSatConfig(PretrainedConfig):
num_conv_pos_embedding_groups (`int`, *optional*, defaults to 16):
Number of groups of 1D convolutional positional embeddings layer.
do_stable_layer_norm (`bool`, *optional*, defaults to `False`):
- Whether to apply *stable* layer norm architecture of the Transformer encoder. `do_stable_layer_norm is True` corresponds to applying layer norm before the attention layer, whereas `do_stable_layer_norm is False` corresponds to applying layer norm after the attention layer.
+ Whether to apply *stable* layer norm architecture of the Transformer encoder. `do_stable_layer_norm is
+ True` corresponds to applying layer norm before the attention layer, whereas `do_stable_layer_norm is
+ False` corresponds to applying layer norm after the attention layer.
apply_spec_augment (`bool`, *optional*, defaults to `True`):
Whether to apply *SpecAugment* data augmentation to the outputs of the feature extractor. For reference see
- [SpecAugment: A Simple Data Augmentation Method for Automatic Speech Recognition](https://arxiv.org/abs/1904.08779).
+ [SpecAugment: A Simple Data Augmentation Method for Automatic Speech
+ Recognition](https://arxiv.org/abs/1904.08779).
mask_time_prob (`float`, *optional*, defaults to 0.05):
Percentage (between 0 and 1) of all feature vectors along the time axis which will be masked. The masking
procecure generates ''mask_time_prob*len(time_axis)/mask_time_length'' independent masks over the axis. If
reasoning from the propability of each feature vector to be chosen as the start of the vector span to be
- masked, *mask_time_prob* should be `prob_vector_start*mask_time_length`. Note that overlap may decrease
- the actual percentage of masked vectors. This is only relevant if `apply_spec_augment is True`.
+ masked, *mask_time_prob* should be `prob_vector_start*mask_time_length`. Note that overlap may decrease the
+ actual percentage of masked vectors. This is only relevant if `apply_spec_augment is True`.
mask_time_length (`int`, *optional*, defaults to 10):
Length of vector span along the time axis.
mask_time_min_masks (`int`, *optional*, defaults to 2),:
- The minimum number of masks of length `mask_feature_length` generated along the time axis, each time
- step, irrespectively of `mask_feature_prob`. Only relevant if
- ''mask_time_prob*len(time_axis)/mask_time_length < mask_time_min_masks''
+ The minimum number of masks of length `mask_feature_length` generated along the time axis, each time step,
+ irrespectively of `mask_feature_prob`. Only relevant if ''mask_time_prob*len(time_axis)/mask_time_length <
+ mask_time_min_masks''
mask_feature_prob (`float`, *optional*, defaults to 0.0):
Percentage (between 0 and 1) of all feature vectors along the feature axis which will be masked. The
masking procecure generates ''mask_feature_prob*len(feature_axis)/mask_time_length'' independent masks over
the axis. If reasoning from the propability of each feature vector to be chosen as the start of the vector
- span to be masked, *mask_feature_prob* should be `prob_vector_start*mask_feature_length`. Note that
- overlap may decrease the actual percentage of masked vectors. This is only relevant if `apply_spec_augment is True`.
+ span to be masked, *mask_feature_prob* should be `prob_vector_start*mask_feature_length`. Note that overlap
+ may decrease the actual percentage of masked vectors. This is only relevant if `apply_spec_augment is
+ True`.
mask_feature_length (`int`, *optional*, defaults to 10):
Length of vector span along the feature axis.
mask_feature_min_masks (`int`, *optional*, defaults to 0),:
@@ -141,9 +145,9 @@ class UniSpeechSatConfig(PretrainedConfig):
Specifies the reduction to apply to the output of `torch.nn.CTCLoss`. Only relevant when training an
instance of [`UniSpeechSatForCTC`].
ctc_zero_infinity (`bool`, *optional*, defaults to `False`):
- Whether to zero infinite losses and the associated gradients of `torch.nn.CTCLoss`. Infinite losses
- mainly occur when the inputs are too short to be aligned to the targets. Only relevant when training an
- instance of [`UniSpeechSatForCTC`].
+ Whether to zero infinite losses and the associated gradients of `torch.nn.CTCLoss`. Infinite losses mainly
+ occur when the inputs are too short to be aligned to the targets. Only relevant when training an instance
+ of [`UniSpeechSatForCTC`].
use_weighted_layer_sum (`bool`, *optional*, defaults to `False`):
Whether to use a weighted average of layer outputs with learned weights. Only relevant when using an
instance of [`UniSpeechSatForSequenceClassification`].
diff --git a/src/transformers/models/unispeech_sat/modeling_unispeech_sat.py b/src/transformers/models/unispeech_sat/modeling_unispeech_sat.py
index 9cdba19745..837ab17a18 100755
--- a/src/transformers/models/unispeech_sat/modeling_unispeech_sat.py
+++ b/src/transformers/models/unispeech_sat/modeling_unispeech_sat.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" PyTorch UniSpeechSat model. """
+""" PyTorch UniSpeechSat model."""
import math
from dataclasses import dataclass
@@ -69,12 +69,13 @@ class UniSpeechSatBaseModelOutput(ModelOutput):
extract_features (`torch.FloatTensor` of shape `(batch_size, sequence_length, conv_dim[-1])`):
Sequence of extracted feature vectors of the last convolutional layer of the model.
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -89,8 +90,7 @@ class UniSpeechSatBaseModelOutput(ModelOutput):
@dataclass
class UniSpeechSatForPreTrainingOutput(ModelOutput):
"""
- Output type of [`UniSpeechSatForPreTrainingOutput`], with potential hidden states and
- attentions.
+ Output type of [`UniSpeechSatForPreTrainingOutput`], with potential hidden states and attentions.
Args:
loss (*optional*, returned when model is in train mode, `torch.FloatTensor` of shape `(1,)`):
@@ -103,12 +103,13 @@ class UniSpeechSatForPreTrainingOutput(ModelOutput):
Quantized extracted feature vectors projected to *config.proj_codevector_dim* representing the positive
target vectors for contrastive loss.
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -136,12 +137,13 @@ class XVectorOutput(ModelOutput):
embeddings (`torch.FloatTensor` of shape `(batch_size, config.xvector_output_dim)`):
Utterance embeddings used for vector similarity-based retrieval.
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -164,8 +166,8 @@ def _compute_mask_indices(
) -> np.ndarray:
"""
Computes random mask spans for a given shape. Used to implement [SpecAugment: A Simple Data Augmentation Method for
- ASR](https://arxiv.org/abs/1904.08779). Note that this method is not optimized to run on TPU and should be run
- on CPU as part of the preprocessing during training.
+ ASR](https://arxiv.org/abs/1904.08779). Note that this method is not optimized to run on TPU and should be run on
+ CPU as part of the preprocessing during training.
Args:
shape: The shape for which to compute masks. This should be of a tuple of size 2 where
@@ -1017,10 +1019,12 @@ class UniSpeechSatPreTrainedModel(PreTrainedModel):
UNISPEECH_SAT_START_DOCSTRING = r"""
- UniSpeechSat was proposed in [wav2vec 2.0: A Framework for Self-Supervised Learning of Speech Representations](https://arxiv.org/abs/2006.11477) by Alexei Baevski, Henry Zhou, Abdelrahman Mohamed, Michael Auli.
+ UniSpeechSat was proposed in [wav2vec 2.0: A Framework for Self-Supervised Learning of Speech
+ Representations](https://arxiv.org/abs/2006.11477) by Alexei Baevski, Henry Zhou, Abdelrahman Mohamed, Michael
+ Auli.
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving etc.).
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving etc.).
This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use
it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
@@ -1029,8 +1033,7 @@ UNISPEECH_SAT_START_DOCSTRING = r"""
Parameters:
config ([`UniSpeechSatConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
@@ -1039,11 +1042,12 @@ UNISPEECH_SAT_INPUTS_DOCSTRING = r"""
input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`):
Float values of input raw speech waveform. Values can be obtained by loading a *.flac* or *.wav* audio file
into an array of type *List[float]* or a *numpy.ndarray*, *e.g.* via the soundfile library (*pip install
- soundfile*). To prepare the array into *input_values*, the [`UniSpeechSatProcessor`]
- should be used for padding and conversion into a tensor of type *torch.FloatTensor*. See
- [`UniSpeechSatProcessor.__call__`] for details.
+ soundfile*). To prepare the array into *input_values*, the [`UniSpeechSatProcessor`] should be used for
+ padding and conversion into a tensor of type *torch.FloatTensor*. See [`UniSpeechSatProcessor.__call__`]
+ for details.
attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0, 1]`:
+ Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0,
+ 1]`:
- 1 for tokens that are **not masked**,
- 0 for tokens that are **masked**.
@@ -1052,12 +1056,12 @@ UNISPEECH_SAT_INPUTS_DOCSTRING = r"""
- `attention_mask` should only be passed if the corresponding processor has
- `config.return_attention_mask == True`. For all models whose processor has
- `config.return_attention_mask == False`, such as [unispeech_sat-base](https://huggingface.co/facebook/unispeech_sat-base-960h), `attention_mask` should **not** be
- passed to avoid degraded performance when doing batched inference. For such models `input_values`
- should simply be padded with 0 and passed without `attention_mask`. Be aware that these models
- also yield slightly different results depending on whether `input_values` is padded or not.
+ `attention_mask` should only be passed if the corresponding processor has `config.return_attention_mask ==
+ True`. For all models whose processor has `config.return_attention_mask == False`, such as
+ [unispeech_sat-base](https://huggingface.co/facebook/unispeech_sat-base-960h), `attention_mask` should
+ **not** be passed to avoid degraded performance when doing batched inference. For such models
+ `input_values` should simply be padded with 0 and passed without `attention_mask`. Be aware that these
+ models also yield slightly different results depending on whether `input_values` is padded or not.
@@ -1196,7 +1200,7 @@ class UniSpeechSatModel(UniSpeechSatPreTrainedModel):
)
-@add_start_docstrings("""UniSpeechSat Model with a quantizer and `VQ` head on top. """, UNISPEECH_SAT_START_DOCSTRING)
+@add_start_docstrings("""UniSpeechSat Model with a quantizer and `VQ` head on top.""", UNISPEECH_SAT_START_DOCSTRING)
class UniSpeechSatForPreTraining(UniSpeechSatPreTrainedModel):
def __init__(self, config: UniSpeechSatConfig):
super().__init__(config)
@@ -1356,7 +1360,7 @@ class UniSpeechSatForPreTraining(UniSpeechSatPreTrainedModel):
@add_start_docstrings(
- """UniSpeechSat Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC). """,
+ """UniSpeechSat Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC).""",
UNISPEECH_SAT_START_DOCSTRING,
)
# Copied from transformers.models.wav2vec2.modeling_wav2vec2.Wav2Vec2ForCTC with Wav2Vec2->UniSpeechSat, wav2vec2->unispeech_sat, WAV_2_VEC_2->UNISPEECH_SAT
@@ -1405,7 +1409,9 @@ class UniSpeechSatForCTC(UniSpeechSatPreTrainedModel):
r"""
labels (`torch.LongTensor` of shape `(batch_size, target_length)`, *optional*):
Labels for connectionist temporal classification. Note that `target_length` has to be smaller or equal to
- the sequence length of the output logits. Indices are selected in `[-100, 0, ..., config.vocab_size - 1]`. All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size - 1]`.
+ the sequence length of the output logits. Indices are selected in `[-100, 0, ..., config.vocab_size - 1]`.
+ All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ...,
+ config.vocab_size - 1]`.
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1520,8 +1526,9 @@ class UniSpeechSatForSequenceClassification(UniSpeechSatPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1622,8 +1629,9 @@ class UniSpeechSatForAudioFrameClassification(UniSpeechSatPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1786,8 +1794,9 @@ class UniSpeechSatForXVector(UniSpeechSatPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
diff --git a/src/transformers/models/vision_encoder_decoder/configuration_vision_encoder_decoder.py b/src/transformers/models/vision_encoder_decoder/configuration_vision_encoder_decoder.py
index 3dc0e4fdf9..72fba3efa8 100644
--- a/src/transformers/models/vision_encoder_decoder/configuration_vision_encoder_decoder.py
+++ b/src/transformers/models/vision_encoder_decoder/configuration_vision_encoder_decoder.py
@@ -27,20 +27,20 @@ logger = logging.get_logger(__name__)
class VisionEncoderDecoderConfig(PretrainedConfig):
r"""
[`VisionEncoderDecoderConfig`] is the configuration class to store the configuration of a
- [`VisionEncoderDecoderModel`]. It is used to instantiate a Vision-Encoder-Text-Decoder model
- according to the specified arguments, defining the encoder and decoder configs.
+ [`VisionEncoderDecoderModel`]. It is used to instantiate a Vision-Encoder-Text-Decoder model according to the
+ specified arguments, defining the encoder and decoder configs.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
kwargs (*optional*):
Dictionary of keyword arguments. Notably:
- - **encoder** ([`PretrainedConfig`], *optional*) -- An instance of a configuration
- object that defines the encoder config.
- - **decoder** ([`PretrainedConfig`], *optional*) -- An instance of a configuration
- object that defines the decoder config.
+ - **encoder** ([`PretrainedConfig`], *optional*) -- An instance of a configuration object that defines
+ the encoder config.
+ - **decoder** ([`PretrainedConfig`], *optional*) -- An instance of a configuration object that defines
+ the decoder config.
Examples:
@@ -95,8 +95,8 @@ class VisionEncoderDecoderConfig(PretrainedConfig):
cls, encoder_config: PretrainedConfig, decoder_config: PretrainedConfig, **kwargs
) -> PretrainedConfig:
r"""
- Instantiate a [`VisionEncoderDecoderConfig`] (or a derived class) from a pre-trained encoder
- model configuration and decoder model configuration.
+ Instantiate a [`VisionEncoderDecoderConfig`] (or a derived class) from a pre-trained encoder model
+ configuration and decoder model configuration.
Returns:
[`VisionEncoderDecoderConfig`]: An instance of a configuration object
diff --git a/src/transformers/models/vision_encoder_decoder/modeling_flax_vision_encoder_decoder.py b/src/transformers/models/vision_encoder_decoder/modeling_flax_vision_encoder_decoder.py
index 35b0629adf..e97a8331f8 100644
--- a/src/transformers/models/vision_encoder_decoder/modeling_flax_vision_encoder_decoder.py
+++ b/src/transformers/models/vision_encoder_decoder/modeling_flax_vision_encoder_decoder.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" Classes to support Vision-Encoder-Text-Decoder architectures """
+""" Classes to support Vision-Encoder-Text-Decoder architectures"""
import os
@@ -41,35 +41,37 @@ _CONFIG_FOR_DOC = "VisionEncoderDecoderConfig"
VISION_ENCODER_DECODER_START_DOCSTRING = r"""
This class can be used to initialize an image-to-text-sequence model with any pretrained vision autoencoding model
as the encoder and any pretrained text autoregressive model as the decoder. The encoder is loaded via
- [`~AutoModel.from_pretrained`] function and the decoder is loaded via
- [`~AutoModelForCausalLM.from_pretrained`] function. Cross-attention layers are automatically added
- to the decoder and should be fine-tuned on a downstream generative task, like image captioning.
+ [`~AutoModel.from_pretrained`] function and the decoder is loaded via [`~AutoModelForCausalLM.from_pretrained`]
+ function. Cross-attention layers are automatically added to the decoder and should be fine-tuned on a downstream
+ generative task, like image captioning.
The effectiveness of initializing sequence-to-sequence models with pretrained checkpoints for sequence generation
- tasks was shown in [Leveraging Pre-trained Checkpoints for Sequence Generation Tasks](https://arxiv.org/abs/1907.12461) by Sascha Rothe, Shashi Narayan, Aliaksei Severyn. Michael Matena, Yanqi
+ tasks was shown in [Leveraging Pre-trained Checkpoints for Sequence Generation
+ Tasks](https://arxiv.org/abs/1907.12461) by Sascha Rothe, Shashi Narayan, Aliaksei Severyn. Michael Matena, Yanqi
Zhou, Wei Li, Peter J. Liu.
- Additionally, in [TrOCR: Transformer-based Optical Character Recognition with Pre-trained Models](https://arxiv.org/abs/2109.10282) it is shown how leveraging large pretrained vision models for optical
+ Additionally, in [TrOCR: Transformer-based Optical Character Recognition with Pre-trained
+ Models](https://arxiv.org/abs/2109.10282) it is shown how leveraging large pretrained vision models for optical
character recognition (OCR) yields a significant performance improvement.
After such a Vision-Encoder-Text-Decoder model has been trained/fine-tuned, it can be saved/loaded just like any
other models (see the examples for more information).
- This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading or saving, resizing the input
- embeddings, pruning heads etc.)
+ This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a Flax Linen [flax.nn.Module](https://flax.readthedocs.io/en/latest/_autosummary/flax.nn.module.html) subclass. Use it as a regular Flax
- Module and refer to the Flax documentation for all matter related to general usage and behavior.
+ This model is also a Flax Linen
+ [flax.nn.Module](https://flax.readthedocs.io/en/latest/_autosummary/flax.nn.module.html) subclass. Use it as a
+ regular Flax Module and refer to the Flax documentation for all matter related to general usage and behavior.
Parameters:
config ([`VisionEncoderDecoderConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the
- model weights.
+ configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the model weights.
dtype (`jax.numpy.dtype`, *optional*, defaults to `jax.numpy.float32`):
- The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on
- GPUs) and `jax.numpy.bfloat16` (on TPUs).
+ The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on GPUs) and
+ `jax.numpy.bfloat16` (on TPUs).
This can be used to enable mixed-precision training or half-precision inference on GPUs or TPUs. If
specified all the computation will be performed with the given `dtype`.
@@ -77,27 +79,25 @@ VISION_ENCODER_DECODER_START_DOCSTRING = r"""
**Note that this only specifies the dtype of the computation and does not influence the dtype of model
parameters.**
- If you wish to change the dtype of the model parameters, see
- [`~FlaxPreTrainedModel.to_fp16`] and [`~FlaxPreTrainedModel.to_bf16`].
+ If you wish to change the dtype of the model parameters, see [`~FlaxPreTrainedModel.to_fp16`] and
+ [`~FlaxPreTrainedModel.to_bf16`].
"""
VISION_ENCODER_DECODER_INPUTS_DOCSTRING = r"""
Args:
pixel_values (`jnp.ndarray` of shape `(batch_size, num_channels, height, width)`):
Pixel values. Pixel values can be obtained using the vision model's feature extractor. For example, using
- [`ViTFeatureExtractor`]. See [`ViTFeatureExtractor.__call__`] for
- details.
+ [`ViTFeatureExtractor`]. See [`ViTFeatureExtractor.__call__`] for details.
decoder_input_ids (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`, *optional*):
Indices of decoder input sequence tokens in the vocabulary.
- Indices can be obtained using [`PreTrainedTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`PreTrainedTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are decoder input IDs?](../glossary#decoder-input-ids)
decoder_attention_mask (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`, *optional*):
- Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will
- also be used by default.
+ Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
+ be used by default.
decoder_position_ids (`jnp.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the
range `[0, config.decoder.max_position_embeddings - 1]`.
@@ -108,16 +108,14 @@ VISION_ENCODER_DECODER_INPUTS_DOCSTRING = r"""
Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
more detail.
return_dict (`bool`, *optional*):
- If set to `True`, the model will return a [`~file_utils.FlaxSeq2SeqLMOutput`] instead
- of a plain tuple.
+ If set to `True`, the model will return a [`~file_utils.FlaxSeq2SeqLMOutput`] instead of a plain tuple.
"""
VISION_ENCODER_DECODER_ENCODE_INPUTS_DOCSTRING = r"""
Args:
pixel_values (`jnp.ndarray` of shape `(batch_size, num_channels, height, width)`):
Pixel values. Pixel values can be obtained using the vision model's feature extractor. For example, using
- [`ViTFeatureExtractor`]. See [`ViTFeatureExtractor.__call__`] for
- details.
+ [`ViTFeatureExtractor`]. See [`ViTFeatureExtractor.__call__`] for details.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -125,8 +123,7 @@ VISION_ENCODER_DECODER_ENCODE_INPUTS_DOCSTRING = r"""
Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
more detail.
return_dict (`bool`, *optional*):
- If set to `True`, the model will return a [`~file_utils.FlaxBaseModelOutput`] instead
- of a plain tuple.
+ If set to `True`, the model will return a [`~file_utils.FlaxBaseModelOutput`] instead of a plain tuple.
"""
VISION_ENCODER_DECODER_DECODE_INPUTS_DOCSTRING = r"""
@@ -134,26 +131,24 @@ VISION_ENCODER_DECODER_DECODE_INPUTS_DOCSTRING = r"""
decoder_input_ids (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`, *optional*):
Indices of decoder input sequence tokens in the vocabulary.
- Indices can be obtained using [`PreTrainedTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`PreTrainedTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are decoder input IDs?](../glossary#decoder-input-ids)
If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
`past_key_values`).
- For sequence to sequence training, `decoder_input_ids` should be provided. If no
- `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to
- the right for denoising pre-training.
+ For sequence to sequence training, `decoder_input_ids` should be provided. If no `decoder_input_ids` is
+ provided, the model will create this tensor by shifting the `input_ids` to the right for denoising
+ pre-training.
encoder_outputs (`tuple(tuple(jnp.ndarray)`):
- Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*:
- `attentions`) `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`,
- *optional*) is a sequence of hidden-states at the output of the last layer of the encoder. Used in the
- cross-attention of the decoder.
+ Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`)
+ `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of
+ hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder.
decoder_attention_mask (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`, *optional*):
- Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will
- also be used by default.
+ Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
+ be used by default.
decoder_position_ids (`jnp.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the
range `[0, config.decoder.max_position_embeddings - 1]`.
@@ -167,8 +162,8 @@ VISION_ENCODER_DECODER_DECODE_INPUTS_DOCSTRING = r"""
Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
more detail.
return_dict (`bool`, *optional*):
- If set to `True`, the model will return a
- [`~file_utils.FlaxCausalLMOutputWithCrossAttentions`] instead of a plain tuple.
+ If set to `True`, the model will return a [`~file_utils.FlaxCausalLMOutputWithCrossAttentions`] instead of
+ a plain tuple.
"""
@@ -272,11 +267,10 @@ class FlaxVisionEncoderDecoderModule(nn.Module):
@add_start_docstrings(VISION_ENCODER_DECODER_START_DOCSTRING)
class FlaxVisionEncoderDecoderModel(FlaxPreTrainedModel):
r"""
- [`FlaxVisionEncoderDecoderModel`] is a generic model class that will be instantiated as a
- transformer architecture with the module (flax.nn.Module) of one of the base vision model classes of the library as
- encoder module and another one as decoder module when created with the
- :meth*~transformers.FlaxAutoModel.from_pretrained* class method for the encoder and
- :meth*~transformers.FlaxAutoModelForCausalLM.from_pretrained* class method for the decoder.
+ [`FlaxVisionEncoderDecoderModel`] is a generic model class that will be instantiated as a transformer architecture
+ with the module (flax.nn.Module) of one of the base vision model classes of the library as encoder module and
+ another one as decoder module when created with the :meth*~transformers.FlaxAutoModel.from_pretrained* class method
+ for the encoder and :meth*~transformers.FlaxAutoModelForCausalLM.from_pretrained* class method for the decoder.
"""
config_class = VisionEncoderDecoderConfig
base_model_prefix = "vision_encoder_decoder"
@@ -349,9 +343,10 @@ class FlaxVisionEncoderDecoderModel(FlaxPreTrainedModel):
maximum possible length for auto-regressive decoding. Defines the sequence length of the initialized
cache.
encoder_outputs (`Union[FlaxBaseModelOutput, tuple(tuple(jnp.ndarray)]`):
- `encoder_outputs` consists of (`last_hidden_state`, *optional*: `hidden_states`,
- *optional*: `attentions`). `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of hidden-states at the output of the last layer of the
- encoder. Used in the cross-attention of the decoder.
+ `encoder_outputs` consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*:
+ `attentions`). `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*)
+ is a sequence of hidden-states at the output of the last layer of the encoder. Used in the
+ cross-attention of the decoder.
"""
# init input variables to retrieve cache
decoder_input_ids = jnp.ones((batch_size, max_length), dtype="i4")
@@ -729,8 +724,8 @@ class FlaxVisionEncoderDecoderModel(FlaxPreTrainedModel):
Information necessary to initiate the decoder. Can be either:
- A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co.
- Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under
- a user or organization name, like `dbmdz/bert-base-german-cased`.
+ Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under a
+ user or organization name, like `dbmdz/bert-base-german-cased`.
- A path to a *directory* containing model weights saved using
[`~FlaxPreTrainedModel.save_pretrained`], e.g., `./my_model_directory/`.
diff --git a/src/transformers/models/vision_encoder_decoder/modeling_vision_encoder_decoder.py b/src/transformers/models/vision_encoder_decoder/modeling_vision_encoder_decoder.py
index 97706bb3f0..23759c543d 100644
--- a/src/transformers/models/vision_encoder_decoder/modeling_vision_encoder_decoder.py
+++ b/src/transformers/models/vision_encoder_decoder/modeling_vision_encoder_decoder.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" Classes to support Vision-Encoder-Text-Decoder architectures """
+""" Classes to support Vision-Encoder-Text-Decoder architectures"""
from typing import Optional
@@ -57,79 +57,79 @@ _CONFIG_FOR_DOC = "VisionEncoderDecoderConfig"
VISION_ENCODER_DECODER_START_DOCSTRING = r"""
This class can be used to initialize an image-to-text-sequence model with any pretrained vision autoencoding model
as the encoder and any pretrained text autoregressive model as the decoder. The encoder is loaded via
- [`~AutoModel.from_pretrained`] function and the decoder is loaded via
- [`~AutoModelForCausalLM.from_pretrained`] function. Cross-attention layers are automatically added
- to the decoder and should be fine-tuned on a downstream generative task, like image captioning.
+ [`~AutoModel.from_pretrained`] function and the decoder is loaded via [`~AutoModelForCausalLM.from_pretrained`]
+ function. Cross-attention layers are automatically added to the decoder and should be fine-tuned on a downstream
+ generative task, like image captioning.
The effectiveness of initializing sequence-to-sequence models with pretrained checkpoints for sequence generation
- tasks was shown in [Leveraging Pre-trained Checkpoints for Sequence Generation Tasks](https://arxiv.org/abs/1907.12461) by Sascha Rothe, Shashi Narayan, Aliaksei Severyn. Michael Matena, Yanqi
+ tasks was shown in [Leveraging Pre-trained Checkpoints for Sequence Generation
+ Tasks](https://arxiv.org/abs/1907.12461) by Sascha Rothe, Shashi Narayan, Aliaksei Severyn. Michael Matena, Yanqi
Zhou, Wei Li, Peter J. Liu.
- Additionally, in [TrOCR: Transformer-based Optical Character Recognition with Pre-trained Models](https://arxiv.org/abs/2109.10282) it is shown how leveraging large pretrained vision models for optical
+ Additionally, in [TrOCR: Transformer-based Optical Character Recognition with Pre-trained
+ Models](https://arxiv.org/abs/2109.10282) it is shown how leveraging large pretrained vision models for optical
character recognition (OCR) yields a significant performance improvement.
After such a Vision-Encoder-Text-Decoder model has been trained/fine-tuned, it can be saved/loaded just like any
other models (see the examples for more information).
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
Parameters:
config ([`VisionEncoderDecoderConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
VISION_ENCODER_DECODER_INPUTS_DOCSTRING = r"""
Args:
pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
Pixel values. Pixel values can be obtained using a feature extractor (e.g. if you use ViT as the encoder,
- you should use [`ViTFeatureExtractor`]). See
- [`ViTFeatureExtractor.__call__`] for details.
+ you should use [`ViTFeatureExtractor`]). See [`ViTFeatureExtractor.__call__`] for details.
decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
Indices of decoder input sequence tokens in the vocabulary.
- Indices can be obtained using [`PreTrainedTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`PreTrainedTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
`past_key_values`).
- For training, `decoder_input_ids` are automatically created by the model by shifting the `labels`
- to the right, replacing -100 by the `pad_token_id` and prepending them with the
- `decoder_start_token_id`.
+ For training, `decoder_input_ids` are automatically created by the model by shifting the `labels` to the
+ right, replacing -100 by the `pad_token_id` and prepending them with the `decoder_start_token_id`.
decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
- Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will
- also be used by default.
+ Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
+ be used by default.
encoder_outputs (`tuple(torch.FloatTensor)`, *optional*):
- This tuple must consist of (`last_hidden_state`, *optional*: `hidden_states`, *optional*:
- `attentions`) `last_hidden_state` (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`) is a tensor of hidden-states at the output of the last layer of the
- encoder. Used in the cross-attention of the decoder.
+ This tuple must consist of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`)
+ `last_hidden_state` (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`) is a tensor
+ of hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the
+ decoder.
past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`):
Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids`
- (those that don't have their past key value states given to this model) of shape `(batch_size, 1)`
- instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`.
+ If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
+ don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
+ `decoder_input_ids` of shape `(batch_size, sequence_length)`.
decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*):
Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded
- representation. This is useful if you want more control over how to convert `decoder_input_ids`
- indices into associated vectors than the model's internal embedding lookup matrix.
+ representation. This is useful if you want more control over how to convert `decoder_input_ids` indices
+ into associated vectors than the model's internal embedding lookup matrix.
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss for the decoder. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
+ Labels for computing the masked language modeling loss for the decoder. Indices should be in `[-100, 0,
+ ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
(masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
use_cache (`bool`, *optional*):
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up
- decoding (see `past_key_values`).
+ If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+ `past_key_values`).
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -137,8 +137,7 @@ VISION_ENCODER_DECODER_INPUTS_DOCSTRING = r"""
Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
more detail.
return_dict (`bool`, *optional*):
- If set to `True`, the model will return a [`~file_utils.Seq2SeqLMOutput`] instead of a
- plain tuple.
+ If set to `True`, the model will return a [`~file_utils.Seq2SeqLMOutput`] instead of a plain tuple.
kwargs: (*optional*) Remaining dictionary of keyword arguments. Keyword arguments come in two flavors:
- Without a prefix which will be input as `**encoder_kwargs` for the encoder forward function.
@@ -149,9 +148,9 @@ VISION_ENCODER_DECODER_INPUTS_DOCSTRING = r"""
@add_start_docstrings(VISION_ENCODER_DECODER_START_DOCSTRING)
class VisionEncoderDecoderModel(PreTrainedModel):
r"""
- [`VisionEncoderDecoderModel`] is a generic model class that will be instantiated as a
- transformer architecture with one of the base vision model classes of the library as encoder and another one as
- decoder when created with the :meth*~transformers.AutoModel.from_pretrained* class method for the encoder and
+ [`VisionEncoderDecoderModel`] is a generic model class that will be instantiated as a transformer architecture with
+ one of the base vision model classes of the library as encoder and another one as decoder when created with the
+ :meth*~transformers.AutoModel.from_pretrained* class method for the encoder and
:meth*~transformers.AutoModelForCausalLM.from_pretrained* class method for the decoder.
"""
config_class = VisionEncoderDecoderConfig
@@ -257,8 +256,8 @@ class VisionEncoderDecoderModel(PreTrainedModel):
checkpoints.
- The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated). To
- train the model, you need to first set it back in training mode with `model.train()`.
+ The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated). To train
+ the model, you need to first set it back in training mode with `model.train()`.
Params:
encoder_pretrained_model_name_or_path (:obj: *str*, *optional*):
@@ -269,22 +268,22 @@ class VisionEncoderDecoderModel(PreTrainedModel):
- A path to a *directory* containing model weights saved using
[`~PreTrainedModel.save_pretrained`], e.g., `./my_model_directory/`.
- A path or url to a *tensorflow index checkpoint file* (e.g, `./tf_model/model.ckpt.index`). In
- this case, `from_tf` should be set to `True` and a configuration object should be provided
- as `config` argument. This loading path is slower than converting the TensorFlow checkpoint in
- a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards.
+ this case, `from_tf` should be set to `True` and a configuration object should be provided as
+ `config` argument. This loading path is slower than converting the TensorFlow checkpoint in a
+ PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards.
decoder_pretrained_model_name_or_path (:obj: *str*, *optional*, defaults to *None*):
Information necessary to initiate the text decoder. Can be either:
- A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co.
- Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under
- a user or organization name, like `dbmdz/bert-base-german-cased`.
+ Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under a
+ user or organization name, like `dbmdz/bert-base-german-cased`.
- A path to a *directory* containing model weights saved using
[`~PreTrainedModel.save_pretrained`], e.g., `./my_model_directory/`.
- A path or url to a *tensorflow index checkpoint file* (e.g, `./tf_model/model.ckpt.index`). In
- this case, `from_tf` should be set to `True` and a configuration object should be provided
- as `config` argument. This loading path is slower than converting the TensorFlow checkpoint in
- a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards.
+ this case, `from_tf` should be set to `True` and a configuration object should be provided as
+ `config` argument. This loading path is slower than converting the TensorFlow checkpoint in a
+ PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards.
model_args (remaining positional arguments, *optional*):
All remaning positional arguments will be passed to the underlying model's `__init__` method.
diff --git a/src/transformers/models/vision_text_dual_encoder/configuration_vision_text_dual_encoder.py b/src/transformers/models/vision_text_dual_encoder/configuration_vision_text_dual_encoder.py
index 7de8e2e494..e8f4d43a8b 100644
--- a/src/transformers/models/vision_text_dual_encoder/configuration_vision_text_dual_encoder.py
+++ b/src/transformers/models/vision_text_dual_encoder/configuration_vision_text_dual_encoder.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" VisionTextDualEncoder model configuration """
+""" VisionTextDualEncoder model configuration"""
import copy
@@ -28,12 +28,11 @@ logger = logging.get_logger(__name__)
class VisionTextDualEncoderConfig(PretrainedConfig):
r"""
[`VisionTextDualEncoderConfig`] is the configuration class to store the configuration of a
- [`VisionTextDualEncoderModel`]. It is used to instantiate
- [`VisionTextDualEncoderModel`] model according to the specified arguments, defining the text
- model and vision model configs.
+ [`VisionTextDualEncoderModel`]. It is used to instantiate [`VisionTextDualEncoderModel`] model according to the
+ specified arguments, defining the text model and vision model configs.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
text_config_dict (`dict`):
@@ -106,8 +105,8 @@ class VisionTextDualEncoderConfig(PretrainedConfig):
@classmethod
def from_vision_text_configs(cls, vision_config: PretrainedConfig, text_config: PretrainedConfig, **kwargs):
r"""
- Instantiate a [`VisionTextDualEncoderConfig`] (or a derived class) from text model configuration and
- vision model configuration.
+ Instantiate a [`VisionTextDualEncoderConfig`] (or a derived class) from text model configuration and vision
+ model configuration.
Returns:
[`VisionTextDualEncoderConfig`]: An instance of a configuration object
@@ -117,8 +116,7 @@ class VisionTextDualEncoderConfig(PretrainedConfig):
def to_dict(self):
"""
- Serializes this instance to a Python dictionary. Override the default
- [`~PretrainedConfig.to_dict`].
+ Serializes this instance to a Python dictionary. Override the default [`~PretrainedConfig.to_dict`].
Returns:
`Dict[str, any]`: Dictionary of all the attributes that make up this configuration instance,
diff --git a/src/transformers/models/vision_text_dual_encoder/modeling_flax_vision_text_dual_encoder.py b/src/transformers/models/vision_text_dual_encoder/modeling_flax_vision_text_dual_encoder.py
index 610741da39..2b54e310b9 100644
--- a/src/transformers/models/vision_text_dual_encoder/modeling_flax_vision_text_dual_encoder.py
+++ b/src/transformers/models/vision_text_dual_encoder/modeling_flax_vision_text_dual_encoder.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" Flax VisionTextDualEncoder model. """
+""" Flax VisionTextDualEncoder model."""
from typing import Optional, Tuple
@@ -38,8 +38,8 @@ _CONFIG_FOR_DOC = "VisionTextDualEncoderConfig"
VISION_TEXT_DUAL_ENCODER_START_DOCSTRING = r"""
This class can be used to initialize a vision-text dual encoder model with any pretrained vision autoencoding model
as the vision encoder and any pretrained text model as the text encoder. The vision and text encoders are loaded
- via the [`~FlaxAutoModel.from_pretrained`] method. The projection layers are automatically added
- to the model and should be fine-tuned on a downstream task, like contrastive image-text modeling.
+ via the [`~FlaxAutoModel.from_pretrained`] method. The projection layers are automatically added to the model and
+ should be fine-tuned on a downstream task, like contrastive image-text modeling.
In [LiT: Zero-Shot Transfer with Locked-image Text Tuning](https://arxiv.org/abs/2111.07991) it is shown how
leveraging pre-trained (locked/frozen) image and text model for contrastive learning yields significant improvment
@@ -48,12 +48,13 @@ VISION_TEXT_DUAL_ENCODER_START_DOCSTRING = r"""
After such a Vision-Text-Dual-Encoder model has been trained/fine-tuned, it can be saved/loaded just like any other
models (see the examples for more information).
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a Flax Linen [flax.linen.Module](https://flax.readthedocs.io/en/latest/flax.linen.html#module) subclass. Use it as a regular Flax linen Module
- and refer to the Flax documentation for all matter related to general usage and behavior.
+ This model is also a Flax Linen [flax.linen.Module](https://flax.readthedocs.io/en/latest/flax.linen.html#module)
+ subclass. Use it as a regular Flax linen Module and refer to the Flax documentation for all matter related to
+ general usage and behavior.
Finally, this model supports inherent JAX features such as:
@@ -65,11 +66,10 @@ VISION_TEXT_DUAL_ENCODER_START_DOCSTRING = r"""
Parameters:
config ([`VisionTextDualEncoderConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the
- model weights.
+ configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the model weights.
dtype (`jax.numpy.dtype`, *optional*, defaults to `jax.numpy.float32`):
- The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on
- GPUs) and `jax.numpy.bfloat16` (on TPUs).
+ The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on GPUs) and
+ `jax.numpy.bfloat16` (on TPUs).
This can be used to enable mixed-precision training or half-precision inference on GPUs or TPUs. If
specified all the computation will be performed with the given `dtype`.
@@ -77,8 +77,8 @@ VISION_TEXT_DUAL_ENCODER_START_DOCSTRING = r"""
**Note that this only specifies the dtype of the computation and does not influence the dtype of model
parameters.**
- If you wish to change the dtype of the model parameters, see
- [`~FlaxPreTrainedModel.to_fp16`] and [`~FlaxPreTrainedModel.to_bf16`].
+ If you wish to change the dtype of the model parameters, see [`~FlaxPreTrainedModel.to_fp16`] and
+ [`~FlaxPreTrainedModel.to_bf16`].
"""
@@ -88,9 +88,8 @@ VISION_TEXT_DUAL_ENCODER_INPUTS_DOCSTRING = r"""
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
it.
- Indices can be obtained using [`PreTrainedTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`PreTrainedTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -101,14 +100,14 @@ VISION_TEXT_DUAL_ENCODER_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
position_ids (`numpy.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using
- a feature extractor (e.g. if you use ViT as the encoder, you should use
- [`ViTFeatureExtractor`]). See [`ViTFeatureExtractor.__call__`] for
- details.
+ a feature extractor (e.g. if you use ViT as the encoder, you should use [`ViTFeatureExtractor`]). See
+ [`ViTFeatureExtractor.__call__`] for details.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -315,15 +314,14 @@ class FlaxVisionTextDualEncoderModel(FlaxPreTrainedModel):
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you
provide it.
- Indices can be obtained using [`PreTrainedTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`]
- for details.
+ Indices can be obtained using [`PreTrainedTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
Returns:
- text_features (`jnp.ndarray` of shape `(batch_size, output_dim`): The text embeddings obtained by
- applying the projection layer to the pooled output of text model.
+ text_features (`jnp.ndarray` of shape `(batch_size, output_dim`): The text embeddings obtained by applying
+ the projection layer to the pooled output of text model.
"""
if position_ids is None:
position_ids = jnp.broadcast_to(jnp.arange(jnp.atleast_2d(input_ids).shape[-1]), input_ids.shape)
@@ -369,12 +367,11 @@ class FlaxVisionTextDualEncoderModel(FlaxPreTrainedModel):
Args:
pixel_values (`numpy.ndarray` of shape `(batch_size, num_channels, height, width)`):
Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained
- using [`ImageFeatureExtractionMixin`]. See
- [`ImageFeatureExtractionMixin.__call__`] for details.
+ using [`ImageFeatureExtractionMixin`]. See [`ImageFeatureExtractionMixin.__call__`] for details.
Returns:
- image_features (`jnp.ndarray` of shape `(batch_size, output_dim`): The image embeddings obtained
- by applying the projection layer to the pooled output of vision model.
+ image_features (`jnp.ndarray` of shape `(batch_size, output_dim`): The image embeddings obtained by
+ applying the projection layer to the pooled output of vision model.
"""
# Handle any PRNG if needed
@@ -410,27 +407,27 @@ class FlaxVisionTextDualEncoderModel(FlaxPreTrainedModel):
Information necessary to initiate the vision model. Can be either:
- A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co.
- Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under
- a user or organization name, like `dbmdz/bert-base-german-cased`.
+ Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under a
+ user or organization name, like `dbmdz/bert-base-german-cased`.
- A path to a *directory* containing model weights saved using
[`~FlaxPreTrainedModel.save_pretrained`], e.g., `./my_model_directory/`.
- A path or url to a *PyTorch checkpoint folder* (e.g, `./pt_model`). In this case, `from_pt`
- should be set to `True` and a configuration object should be provided as `config`
- argument. This loading path is slower than converting the PyTorch checkpoint in a Flax model
- using the provided conversion scripts and loading the Flax model afterwards.
+ should be set to `True` and a configuration object should be provided as `config` argument. This
+ loading path is slower than converting the PyTorch checkpoint in a Flax model using the provided
+ conversion scripts and loading the Flax model afterwards.
text_model_name_or_path (:obj: *str*, *optional*):
Information necessary to initiate the text model. Can be either:
- A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co.
- Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under
- a user or organization name, like `dbmdz/bert-base-german-cased`.
+ Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under a
+ user or organization name, like `dbmdz/bert-base-german-cased`.
- A path to a *directory* containing model weights saved using
[`~FlaxPreTrainedModel.save_pretrained`], e.g., `./my_model_directory/`.
- A path or url to a *PyTorch checkpoint folder* (e.g, `./pt_model`). In this case, `from_pt`
- should be set to `True` and a configuration object should be provided as `config`
- argument. This loading path is slower than converting the PyTorch checkpoint in a Flax model
- using the provided conversion scripts and loading the Flax model afterwards.
+ should be set to `True` and a configuration object should be provided as `config` argument. This
+ loading path is slower than converting the PyTorch checkpoint in a Flax model using the provided
+ conversion scripts and loading the Flax model afterwards.
model_args (remaining positional arguments, *optional*):
All remaning positional arguments will be passed to the underlying model's `__init__` method.
diff --git a/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py b/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py
index 008af3e471..795af8be4c 100755
--- a/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py
+++ b/src/transformers/models/vision_text_dual_encoder/modeling_vision_text_dual_encoder.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" PyTorch VisionTextDualEncoder model. """
+""" PyTorch VisionTextDualEncoder model."""
from typing import Optional
@@ -36,8 +36,8 @@ _CONFIG_FOR_DOC = "VisionTextDualEncoderConfig"
VISION_TEXT_DUAL_ENCODER_START_DOCSTRING = r"""
This class can be used to initialize a vision-text dual encoder model with any pretrained vision autoencoding model
as the vision encoder and any pretrained text model as the text encoder. The vision and text encoders are loaded
- via the [`~AutoModel.from_pretrained`] method. The projection layers are automatically added to
- the model and should be fine-tuned on a downstream task, like contrastive image-text modeling.
+ via the [`~AutoModel.from_pretrained`] method. The projection layers are automatically added to the model and
+ should be fine-tuned on a downstream task, like contrastive image-text modeling.
In [LiT: Zero-Shot Transfer with Locked-image Text Tuning](https://arxiv.org/abs/2111.07991) it is shown how
leveraging pre-trained (locked/frozen) image and text model for contrastive learning yields significant improvment
@@ -46,19 +46,18 @@ VISION_TEXT_DUAL_ENCODER_START_DOCSTRING = r"""
After such a Vision-Text-Dual-Encoder model has been trained/fine-tuned, it can be saved/loaded just like any other
models (see the examples for more information).
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
Parameters:
config ([`VisionEncoderDecoderConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
@@ -68,9 +67,8 @@ VISION_TEXT_DUAL_ENCODER_TEXT_INPUTS_DOCSTRING = r"""
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
it.
- Indices can be obtained using [`PreTrainedTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`PreTrainedTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -81,7 +79,8 @@ VISION_TEXT_DUAL_ENCODER_TEXT_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
output_attentions (`bool`, *optional*):
@@ -98,8 +97,7 @@ VISION_TEXT_DUAL_ENCODER_VISION_INPUTS_DOCSTRING = r"""
Args:
pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using
- [`CLIPFeatureExtractor`]. See [`CLIPFeatureExtractor.__call__`] for
- details.
+ [`CLIPFeatureExtractor`]. See [`CLIPFeatureExtractor.__call__`] for details.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -116,9 +114,8 @@ VISION_TEXT_DUAL_ENCODER_INPUTS_DOCSTRING = r"""
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
it.
- Indices can be obtained using [`CLIPTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`CLIPTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
@@ -129,14 +126,14 @@ VISION_TEXT_DUAL_ENCODER_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using
- a feature extractor (e.g. if you use ViT as the encoder, you should use
- [`ViTFeatureExtractor`]). See [`ViTFeatureExtractor.__call__`] for
- details.
+ a feature extractor (e.g. if you use ViT as the encoder, you should use [`ViTFeatureExtractor`]). See
+ [`ViTFeatureExtractor.__call__`] for details.
return_loss (`bool`, *optional*):
Whether or not to return the contrastive loss.
output_attentions (`bool`, *optional*):
@@ -223,8 +220,8 @@ class VisionTextDualEncoderModel(PreTrainedModel):
):
r"""
Returns:
- text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings
- obtained by applying the projection layer to the pooled output of [`CLIPTextModel`].
+ text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by
+ applying the projection layer to the pooled output of [`CLIPTextModel`].
Examples:
@@ -261,8 +258,8 @@ class VisionTextDualEncoderModel(PreTrainedModel):
):
r"""
Returns:
- image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings
- obtained by applying the projection layer to the pooled output of [`CLIPVisionModel`].
+ image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by
+ applying the projection layer to the pooled output of [`CLIPVisionModel`].
Examples:
@@ -411,27 +408,27 @@ class VisionTextDualEncoderModel(PreTrainedModel):
Information necessary to initiate the vision model. Can be either:
- A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co.
- Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under
- a user or organization name, like `dbmdz/bert-base-german-cased`.
+ Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under a
+ user or organization name, like `dbmdz/bert-base-german-cased`.
- A path to a *directory* containing model weights saved using
[`~FlaxPreTrainedModel.save_pretrained`], e.g., `./my_model_directory/`.
- A path or url to a *PyTorch checkpoint folder* (e.g, `./pt_model`). In this case, `from_pt`
- should be set to `True` and a configuration object should be provided as `config`
- argument. This loading path is slower than converting the PyTorch checkpoint in a Flax model
- using the provided conversion scripts and loading the Flax model afterwards.
+ should be set to `True` and a configuration object should be provided as `config` argument. This
+ loading path is slower than converting the PyTorch checkpoint in a Flax model using the provided
+ conversion scripts and loading the Flax model afterwards.
text_model_name_or_path (:obj: *str*, *optional*):
Information necessary to initiate the text model. Can be either:
- A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co.
- Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under
- a user or organization name, like `dbmdz/bert-base-german-cased`.
+ Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under a
+ user or organization name, like `dbmdz/bert-base-german-cased`.
- A path to a *directory* containing model weights saved using
[`~FlaxPreTrainedModel.save_pretrained`], e.g., `./my_model_directory/`.
- A path or url to a *PyTorch checkpoint folder* (e.g, `./pt_model`). In this case, `from_pt`
- should be set to `True` and a configuration object should be provided as `config`
- argument. This loading path is slower than converting the PyTorch checkpoint in a Flax model
- using the provided conversion scripts and loading the Flax model afterwards.
+ should be set to `True` and a configuration object should be provided as `config` argument. This
+ loading path is slower than converting the PyTorch checkpoint in a Flax model using the provided
+ conversion scripts and loading the Flax model afterwards.
model_args (remaining positional arguments, *optional*):
All remaning positional arguments will be passed to the underlying model's `__init__` method.
diff --git a/src/transformers/models/vision_text_dual_encoder/processing_vision_text_dual_encoder.py b/src/transformers/models/vision_text_dual_encoder/processing_vision_text_dual_encoder.py
index 5e607eb108..b421836f52 100644
--- a/src/transformers/models/vision_text_dual_encoder/processing_vision_text_dual_encoder.py
+++ b/src/transformers/models/vision_text_dual_encoder/processing_vision_text_dual_encoder.py
@@ -30,9 +30,8 @@ class VisionTextDualEncoderProcessor:
Constructs a VisionTextDualEncoder processor which wraps a vision feature extractor and a tokenizer into a single
processor.
- [`VisionTextDualEncoderProcessor`] offers all the functionalities of
- [`AutoFeatureExtractor`] and [`AutoTokenizer`]. See the
- [`~VisionTextDualEncoderProcessor.__call__`] and
+ [`VisionTextDualEncoderProcessor`] offers all the functionalities of [`AutoFeatureExtractor`] and
+ [`AutoTokenizer`]. See the [`~VisionTextDualEncoderProcessor.__call__`] and
[`~VisionTextDualEncoderProcessor.decode`] for more information.
Args:
@@ -67,8 +66,8 @@ class VisionTextDualEncoderProcessor:
This class method is simply calling [`~PreTrainedFeatureExtractor.save_pretrained`] and
- [`~tokenization_utils_base.PreTrainedTokenizer.save_pretrained`]. Please refer to the
- docstrings of the methods above for more information.
+ [`~tokenization_utils_base.PreTrainedTokenizer.save_pretrained`]. Please refer to the docstrings of the methods
+ above for more information.
@@ -84,14 +83,12 @@ class VisionTextDualEncoderProcessor:
@classmethod
def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
r"""
- Instantiate a [`VisionTextDualEncoderProcessor`] from a pretrained VisionTextDualEncoder
- processor.
+ Instantiate a [`VisionTextDualEncoderProcessor`] from a pretrained VisionTextDualEncoder processor.
- This class method is simply calling AutoFeatureExtractor's
- [`~PreTrainedFeatureExtractor.from_pretrained`] and AutoTokenizer's
- [`~tokenization_utils_base.PreTrainedTokenizer.from_pretrained`]. Please refer to the
+ This class method is simply calling AutoFeatureExtractor's [`~PreTrainedFeatureExtractor.from_pretrained`] and
+ AutoTokenizer's [`~tokenization_utils_base.PreTrainedTokenizer.from_pretrained`]. Please refer to the
docstrings of the methods above for more information.
@@ -104,8 +101,7 @@ class VisionTextDualEncoderProcessor:
huggingface.co. Valid model ids can be located at the root-level, like `bert-base-uncased`, or
namespaced under a user or organization name, like `dbmdz/bert-base-german-cased`.
- a path to a *directory* containing a feature extractor file saved using the
- [`~PreTrainedFeatureExtractor.save_pretrained`] method, e.g.,
- `./my_model_directory/`.
+ [`~PreTrainedFeatureExtractor.save_pretrained`] method, e.g., `./my_model_directory/`.
- a path or url to a saved feature extractor JSON *file*, e.g.,
`./my_model_directory/preprocessor_config.json`.
@@ -120,12 +116,11 @@ class VisionTextDualEncoderProcessor:
def __call__(self, text=None, images=None, return_tensors=None, **kwargs):
"""
- Main method to prepare for the model one or several sequences(s) and image(s). This method forwards the
- `text` and `kwargs` arguments to VisionTextDualEncoderTokenizer's
- [`~PreTrainedTokenizer.__call__`] if `text` is not `None` to encode the text. To
- prepare the image(s), this method forwards the `images` and `kwrags` arguments to
- AutoFeatureExtractor's [`~AutoFeatureExtractor.__call__`] if `images` is not `None`.
- Please refer to the doctsring of the above two methods for more information.
+ Main method to prepare for the model one or several sequences(s) and image(s). This method forwards the `text`
+ and `kwargs` arguments to VisionTextDualEncoderTokenizer's [`~PreTrainedTokenizer.__call__`] if `text` is not
+ `None` to encode the text. To prepare the image(s), this method forwards the `images` and `kwrags` arguments to
+ AutoFeatureExtractor's [`~AutoFeatureExtractor.__call__`] if `images` is not `None`. Please refer to the
+ doctsring of the above two methods for more information.
Args:
text (`str`, `List[str]`, `List[List[str]]`):
@@ -150,8 +145,8 @@ class VisionTextDualEncoderProcessor:
- **input_ids** -- List of token ids to be fed to a model. Returned when `text` is not `None`.
- **attention_mask** -- List of indices specifying which tokens should be attended to by the model (when
- `return_attention_mask=True` or if *"attention_mask"* is in `self.model_input_names` and if
- `text` is not `None`).
+ `return_attention_mask=True` or if *"attention_mask"* is in `self.model_input_names` and if `text` is not
+ `None`).
- **pixel_values** -- Pixel values to be fed to a model. Returned when `images` is not `None`.
"""
@@ -175,15 +170,13 @@ class VisionTextDualEncoderProcessor:
def batch_decode(self, *args, **kwargs):
"""
This method forwards all its arguments to VisionTextDualEncoderTokenizer's
- [`~PreTrainedTokenizer.batch_decode`]. Please refer to the docstring of this method for more
- information.
+ [`~PreTrainedTokenizer.batch_decode`]. Please refer to the docstring of this method for more information.
"""
return self.tokenizer.batch_decode(*args, **kwargs)
def decode(self, *args, **kwargs):
"""
- This method forwards all its arguments to VisionTextDualEncoderTokenizer's
- [`~PreTrainedTokenizer.decode`]. Please refer to the docstring of this method for more
- information.
+ This method forwards all its arguments to VisionTextDualEncoderTokenizer's [`~PreTrainedTokenizer.decode`].
+ Please refer to the docstring of this method for more information.
"""
return self.tokenizer.decode(*args, **kwargs)
diff --git a/src/transformers/models/visual_bert/configuration_visual_bert.py b/src/transformers/models/visual_bert/configuration_visual_bert.py
index 5963af26aa..b8aafa2c4d 100644
--- a/src/transformers/models/visual_bert/configuration_visual_bert.py
+++ b/src/transformers/models/visual_bert/configuration_visual_bert.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" VisualBERT model configuration """
+""" VisualBERT model configuration"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging
@@ -36,21 +36,21 @@ VISUAL_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class VisualBertConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`VisualBertModel`]. It is used
- to instantiate an VisualBERT model according to the specified arguments, defining the model architecture.
- Instantiating a configuration with the defaults will yield a similar configuration to that of the VisualBERT
+ This is the configuration class to store the configuration of a [`VisualBertModel`]. It is used to instantiate an
+ VisualBERT model according to the specified arguments, defining the model architecture. Instantiating a
+ configuration with the defaults will yield a similar configuration to that of the VisualBERT
[visualbert-vqa-coco-pre](https://huggingface.co/uclanlp/visualbert-vqa-coco-pre) architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
vocab_size (`int`, *optional*, defaults to 30522):
Vocabulary size of the VisualBERT model. Defines the number of different tokens that can be represented by
- the `inputs_ids` passed when calling [`VisualBertModel`]. Vocabulary size of the
- model. Defines the different tokens that can be represented by the `inputs_ids` passed to the forward
- method of [`VisualBertModel`].
+ the `inputs_ids` passed when calling [`VisualBertModel`]. Vocabulary size of the model. Defines the
+ different tokens that can be represented by the `inputs_ids` passed to the forward method of
+ [`VisualBertModel`].
hidden_size (`int`, *optional*, defaults to 768):
Dimensionality of the encoder layers and the pooler layer.
visual_embedding_dim (`int`, *optional*, defaults to 512):
@@ -62,8 +62,8 @@ class VisualBertConfig(PretrainedConfig):
intermediate_size (`int`, *optional*, defaults to 3072):
Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"selu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"selu"` and `"gelu_new"` are supported.
hidden_dropout_prob (`float`, *optional*, defaults to 0.1):
The dropout probabilitiy for all fully connected layers in the embeddings, encoder, and pooler.
attention_probs_dropout_prob (`float`, *optional*, defaults to 0.1):
@@ -72,20 +72,19 @@ class VisualBertConfig(PretrainedConfig):
The maximum sequence length that this model might ever be used with. Typically set this to something large
just in case (e.g., 512 or 1024 or 2048).
type_vocab_size (`int`, *optional*, defaults to 2):
- The vocabulary size of the `token_type_ids` passed when calling
- [`VisualBertModel`].
+ The vocabulary size of the `token_type_ids` passed when calling [`VisualBertModel`].
initializer_range (`float`, *optional*, defaults to 0.02):
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
layer_norm_eps (`float`, *optional*, defaults to 1e-12):
The epsilon used by the layer normalization layers.
bypass_transformer (`bool`, *optional*, defaults to `False`):
- Whether or not the model should bypass the transformer for the visual embeddings. If set to `True`,
- the model directly concatenates the visual embeddings from [`VisualBertEmbeddings`] with
- text output from transformers, and then pass it to a self-attention layer.
+ Whether or not the model should bypass the transformer for the visual embeddings. If set to `True`, the
+ model directly concatenates the visual embeddings from [`VisualBertEmbeddings`] with text output from
+ transformers, and then pass it to a self-attention layer.
special_visual_initialize (`bool`, *optional*, defaults to `True`):
Whether or not the visual token type and position type embedding weights should be initialized the same as
- the textual token type and positive type embeddings. When set to `True`, the weights of the textual
- token type and position type embeddings are copied to the respective visual embedding layers.
+ the textual token type and positive type embeddings. When set to `True`, the weights of the textual token
+ type and position type embeddings are copied to the respective visual embedding layers.
Example:
diff --git a/src/transformers/models/visual_bert/modeling_visual_bert.py b/src/transformers/models/visual_bert/modeling_visual_bert.py
index 7a5719e299..cb388c60a1 100755
--- a/src/transformers/models/visual_bert/modeling_visual_bert.py
+++ b/src/transformers/models/visual_bert/modeling_visual_bert.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" PyTorch VisualBERT model. """
+""" PyTorch VisualBERT model."""
import math
@@ -569,12 +569,13 @@ class VisualBertForPreTrainingOutput(ModelOutput):
Prediction scores of the sentence-image prediction (classification) head (scores of True/False continuation
before SoftMax).
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -588,19 +589,18 @@ class VisualBertForPreTrainingOutput(ModelOutput):
VISUAL_BERT_START_DOCSTRING = r"""
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
Parameters:
config ([`VisualBertConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
VISUAL_BERT_INPUTS_DOCSTRING = r"""
@@ -608,9 +608,8 @@ VISUAL_BERT_INPUTS_DOCSTRING = r"""
input_ids (`torch.LongTensor` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`BertTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`BertTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
@@ -621,14 +620,16 @@ VISUAL_BERT_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
@@ -638,9 +639,9 @@ VISUAL_BERT_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
visual_embeds (`torch.FloatTensor` of shape `(batch_size, visual_seq_length, visual_embedding_dim)`, *optional*):
The embedded representation of the visual inputs, generally derived using using an object detector.
@@ -908,11 +909,12 @@ class VisualBertForPreTraining(VisualBertPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size, total_sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
+ loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
sentence_image_labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sentence-image prediction (classification) loss. Input should be a sequence
- pair (see `input_ids` docstring) Indices should be in `[0, 1]`:
+ Labels for computing the sentence-image prediction (classification) loss. Input should be a sequence pair
+ (see `input_ids` docstring) Indices should be in `[0, 1]`:
- 0 indicates sequence B is a matching pair of sequence A for the given image,
- 1 indicates sequence B is a random sequence w.r.t A for the given image.
@@ -1048,8 +1050,9 @@ class VisualBertForMultipleChoice(VisualBertPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors.
- (See `input_ids` above)
+ Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
+ num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
+ `input_ids` above)
Returns:
@@ -1197,7 +1200,8 @@ class VisualBertForQuestionAnswering(VisualBertPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size, total_sequence_length)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. A KLDivLoss is computed between the labels and the returned logits.
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. A KLDivLoss is computed between the labels and the returned logits.
Returns:
@@ -1320,7 +1324,8 @@ class VisualBertForVisualReasoning(VisualBertPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. A classification loss is computed (Cross-Entropy) against these labels.
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. A classification loss is computed (Cross-Entropy) against these labels.
Returns:
@@ -1481,8 +1486,8 @@ class VisualBertForRegionToPhraseAlignment(VisualBertPreTrainedModel):
The positions depicting the position of the image embedding corresponding to the textual tokens.
labels (`torch.LongTensor` of shape `(batch_size, total_sequence_length, visual_sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. KLDivLoss is computed against these labels and
- the outputs from the attention layer.
+ Labels for computing the masked language modeling loss. KLDivLoss is computed against these labels and the
+ outputs from the attention layer.
Returns:
diff --git a/src/transformers/models/vit/configuration_vit.py b/src/transformers/models/vit/configuration_vit.py
index 9c1ee38b2e..cb690324bd 100644
--- a/src/transformers/models/vit/configuration_vit.py
+++ b/src/transformers/models/vit/configuration_vit.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" ViT model configuration """
+""" ViT model configuration"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging
@@ -28,12 +28,13 @@ VIT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class ViTConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`ViTModel`]. It is used to
- instantiate an ViT model according to the specified arguments, defining the model architecture. Instantiating a
- configuration with the defaults will yield a similar configuration to that of the ViT [google/vit-base-patch16-224](https://huggingface.co/google/vit-base-patch16-224) architecture.
+ This is the configuration class to store the configuration of a [`ViTModel`]. It is used to instantiate an ViT
+ model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
+ defaults will yield a similar configuration to that of the ViT
+ [google/vit-base-patch16-224](https://huggingface.co/google/vit-base-patch16-224) architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
@@ -46,8 +47,8 @@ class ViTConfig(PretrainedConfig):
intermediate_size (`int`, *optional*, defaults to 3072):
Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"selu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"selu"` and `"gelu_new"` are supported.
hidden_dropout_prob (`float`, *optional*, defaults to 0.1):
The dropout probabilitiy for all fully connected layers in the embeddings, encoder, and pooler.
attention_probs_dropout_prob (`float`, *optional*, defaults to 0.1):
diff --git a/src/transformers/models/vit/feature_extraction_vit.py b/src/transformers/models/vit/feature_extraction_vit.py
index d67f60ef23..fe0f0d824c 100644
--- a/src/transformers/models/vit/feature_extraction_vit.py
+++ b/src/transformers/models/vit/feature_extraction_vit.py
@@ -38,20 +38,20 @@ class ViTFeatureExtractor(FeatureExtractionMixin, ImageFeatureExtractionMixin):
r"""
Constructs a ViT feature extractor.
- This feature extractor inherits from [`FeatureExtractionMixin`] which contains most of the main
- methods. Users should refer to this superclass for more information regarding those methods.
+ This feature extractor inherits from [`FeatureExtractionMixin`] which contains most of the main methods. Users
+ should refer to this superclass for more information regarding those methods.
Args:
do_resize (`bool`, *optional*, defaults to `True`):
Whether to resize the input to a certain `size`.
size (`int` or `Tuple(int)`, *optional*, defaults to 224):
Resize the input to the given size. If a tuple is provided, it should be (width, height). If only an
- integer is provided, then the input will be resized to (size, size). Only has an effect if `do_resize`
- is set to `True`.
+ integer is provided, then the input will be resized to (size, size). Only has an effect if `do_resize` is
+ set to `True`.
resample (`int`, *optional*, defaults to `PIL.Image.BILINEAR`):
An optional resampling filter. This can be one of `PIL.Image.NEAREST`, `PIL.Image.BOX`,
- `PIL.Image.BILINEAR`, `PIL.Image.HAMMING`, `PIL.Image.BICUBIC` or `PIL.Image.LANCZOS`.
- Only has an effect if `do_resize` is set to `True`.
+ `PIL.Image.BILINEAR`, `PIL.Image.HAMMING`, `PIL.Image.BICUBIC` or `PIL.Image.LANCZOS`. Only has an effect
+ if `do_resize` is set to `True`.
do_normalize (`bool`, *optional*, defaults to `True`):
Whether or not to normalize the input with mean and standard deviation.
image_mean (`List[int]`, defaults to `[0.5, 0.5, 0.5]`):
diff --git a/src/transformers/models/vit/modeling_flax_vit.py b/src/transformers/models/vit/modeling_flax_vit.py
index e478063dbb..ace49edac6 100644
--- a/src/transformers/models/vit/modeling_flax_vit.py
+++ b/src/transformers/models/vit/modeling_flax_vit.py
@@ -34,12 +34,12 @@ from .configuration_vit import ViTConfig
VIT_START_DOCSTRING = r"""
- This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading, saving and converting weights from
- PyTorch models)
+ This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading, saving and converting weights from PyTorch models)
- This model is also a Flax Linen [flax.linen.Module](https://flax.readthedocs.io/en/latest/flax.linen.html#module) subclass. Use it as a regular Flax linen Module
- and refer to the Flax documentation for all matter related to general usage and behavior.
+ This model is also a Flax Linen [flax.linen.Module](https://flax.readthedocs.io/en/latest/flax.linen.html#module)
+ subclass. Use it as a regular Flax linen Module and refer to the Flax documentation for all matter related to
+ general usage and behavior.
Finally, this model supports inherent JAX features such as:
@@ -51,11 +51,10 @@ VIT_START_DOCSTRING = r"""
Parameters:
config ([`ViTConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the
- model weights.
+ configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the model weights.
dtype (`jax.numpy.dtype`, *optional*, defaults to `jax.numpy.float32`):
- The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on
- GPUs) and `jax.numpy.bfloat16` (on TPUs).
+ The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on GPUs) and
+ `jax.numpy.bfloat16` (on TPUs).
This can be used to enable mixed-precision training or half-precision inference on GPUs or TPUs. If
specified all the computation will be performed with the given `dtype`.
@@ -63,8 +62,8 @@ VIT_START_DOCSTRING = r"""
**Note that this only specifies the dtype of the computation and does not influence the dtype of model
parameters.**
- If you wish to change the dtype of the model parameters, see
- [`~FlaxPreTrainedModel.to_fp16`] and [`~FlaxPreTrainedModel.to_bf16`].
+ If you wish to change the dtype of the model parameters, see [`~FlaxPreTrainedModel.to_fp16`] and
+ [`~FlaxPreTrainedModel.to_bf16`].
"""
VIT_INPUTS_DOCSTRING = r"""
diff --git a/src/transformers/models/vit/modeling_tf_vit.py b/src/transformers/models/vit/modeling_tf_vit.py
index f04169562e..24cfd4d3a4 100644
--- a/src/transformers/models/vit/modeling_tf_vit.py
+++ b/src/transformers/models/vit/modeling_tf_vit.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" TF 2.0 ViT model. """
+""" TF 2.0 ViT model."""
import collections.abc
@@ -592,13 +592,13 @@ class TFViTPreTrainedModel(TFPreTrainedModel):
VIT_START_DOCSTRING = r"""
- This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading or saving, resizing the input
- embeddings, pruning heads etc.)
+ This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use
- it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage
- and behavior.
+ This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it
+ as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and
+ behavior.
@@ -607,21 +607,22 @@ VIT_START_DOCSTRING = r"""
- having all inputs as keyword arguments (like PyTorch models), or
- having all inputs as a list, tuple or dict in the first positional arguments.
- This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all
- the tensors in the first argument of the model call function: `model(inputs)`.
+ This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all the
+ tensors in the first argument of the model call function: `model(inputs)`.
Args:
config ([`ViTConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~TFPreTrainedModel.from_pretrained`] method to load the
- model weights.
+ configuration. Check out the [`~TFPreTrainedModel.from_pretrained`] method to load the model weights.
"""
VIT_INPUTS_DOCSTRING = r"""
Args:
- pixel_values (`np.ndarray`, `tf.Tensor`, `List[tf.Tensor]` ``Dict[str, tf.Tensor]` or `Dict[str, np.ndarray]` and each example must have the shape `(batch_size, num_channels, height, width)`): Pixel values. Pixel values can be obtained using [`ViTFeatureExtractor`]. See [`ViTFeatureExtractor.__call__`] for details.
+ pixel_values (`np.ndarray`, `tf.Tensor`, `List[tf.Tensor]` ``Dict[str, tf.Tensor]` or `Dict[str, np.ndarray]` and each example must have the shape `(batch_size, num_channels, height, width)`):
+ Pixel values. Pixel values can be obtained using [`ViTFeatureExtractor`]. See
+ [`ViTFeatureExtractor.__call__`] for details.
head_mask (`np.ndarray` or `tf.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:
@@ -640,8 +641,8 @@ VIT_INPUTS_DOCSTRING = r"""
interpolate_pos_encoding (`bool`, *optional*):
Whether to interpolate the pre-trained position encodings.
return_dict (`bool`, *optional*):
- Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This
- argument can be used in eager mode, in graph mode the value will always be set to True.
+ Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This argument can be used
+ in eager mode, in graph mode the value will always be set to True.
training (`bool`, *optional*, defaults to `False``):
Whether or not to use the model in training mode (some modules like dropout modules have different
behaviors between training and evaluation).
@@ -788,8 +789,9 @@ class TFViTForImageClassification(TFViTPreTrainedModel, TFSequenceClassification
) -> Union[TFSequenceClassifierOutput, Tuple[tf.Tensor]]:
r"""
labels (`tf.Tensor` or `np.ndarray` of shape `(batch_size,)`, *optional*):
- Labels for computing the image classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
Returns:
diff --git a/src/transformers/models/vit/modeling_vit.py b/src/transformers/models/vit/modeling_vit.py
index 0011929a5c..f8569b9645 100644
--- a/src/transformers/models/vit/modeling_vit.py
+++ b/src/transformers/models/vit/modeling_vit.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" PyTorch ViT model. """
+""" PyTorch ViT model."""
import collections.abc
@@ -437,15 +437,14 @@ class ViTPreTrainedModel(PreTrainedModel):
VIT_START_DOCSTRING = r"""
- This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use
- it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
+ This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it
+ as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
behavior.
Parameters:
config ([`ViTConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
VIT_INPUTS_DOCSTRING = r"""
@@ -622,8 +621,9 @@ class ViTForImageClassification(ViTPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the image classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
Returns:
diff --git a/src/transformers/models/wav2vec2/configuration_wav2vec2.py b/src/transformers/models/wav2vec2/configuration_wav2vec2.py
index a9d47cc632..a948f62c90 100644
--- a/src/transformers/models/wav2vec2/configuration_wav2vec2.py
+++ b/src/transformers/models/wav2vec2/configuration_wav2vec2.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" Wav2Vec2 model configuration """
+""" Wav2Vec2 model configuration"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging
@@ -28,21 +28,21 @@ WAV_2_VEC_2_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class Wav2Vec2Config(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`Wav2Vec2Model`]. It is used to
- instantiate an Wav2Vec2 model according to the specified arguments, defining the model architecture. Instantiating
- a configuration with the defaults will yield a similar configuration to that of the Wav2Vec2
+ This is the configuration class to store the configuration of a [`Wav2Vec2Model`]. It is used to instantiate an
+ Wav2Vec2 model according to the specified arguments, defining the model architecture. Instantiating a configuration
+ with the defaults will yield a similar configuration to that of the Wav2Vec2
[facebook/wav2vec2-base-960h](https://huggingface.co/facebook/wav2vec2-base-960h) architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
vocab_size (`int`, *optional*, defaults to 32):
Vocabulary size of the Wav2Vec2 model. Defines the number of different tokens that can be represented by
- the `inputs_ids` passed when calling [`Wav2Vec2Model`] or
- [`TFWav2Vec2Model`]. Vocabulary size of the model. Defines the different tokens that can
- be represented by the *inputs_ids* passed to the forward method of [`Wav2Vec2Model`].
+ the `inputs_ids` passed when calling [`Wav2Vec2Model`] or [`TFWav2Vec2Model`]. Vocabulary size of the
+ model. Defines the different tokens that can be represented by the *inputs_ids* passed to the forward
+ method of [`Wav2Vec2Model`].
hidden_size (`int`, *optional*, defaults to 768):
Dimensionality of the encoder layers and the pooler layer.
num_hidden_layers (`int`, *optional*, defaults to 12):
@@ -52,8 +52,8 @@ class Wav2Vec2Config(PretrainedConfig):
intermediate_size (`int`, *optional*, defaults to 3072):
Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"selu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"selu"` and `"gelu_new"` are supported.
hidden_dropout (`float`, *optional*, defaults to 0.1):
The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
attention_dropout (`float`, *optional*, defaults to 0.1):
@@ -93,28 +93,32 @@ class Wav2Vec2Config(PretrainedConfig):
num_conv_pos_embedding_groups (`int`, *optional*, defaults to 16):
Number of groups of 1D convolutional positional embeddings layer.
do_stable_layer_norm (`bool`, *optional*, defaults to `False`):
- Whether to apply *stable* layer norm architecture of the Transformer encoder. `do_stable_layer_norm is True` corresponds to applying layer norm before the attention layer, whereas `do_stable_layer_norm is False` corresponds to applying layer norm after the attention layer.
+ Whether to apply *stable* layer norm architecture of the Transformer encoder. `do_stable_layer_norm is
+ True` corresponds to applying layer norm before the attention layer, whereas `do_stable_layer_norm is
+ False` corresponds to applying layer norm after the attention layer.
apply_spec_augment (`bool`, *optional*, defaults to `True`):
Whether to apply *SpecAugment* data augmentation to the outputs of the feature extractor. For reference see
- [SpecAugment: A Simple Data Augmentation Method for Automatic Speech Recognition](https://arxiv.org/abs/1904.08779).
+ [SpecAugment: A Simple Data Augmentation Method for Automatic Speech
+ Recognition](https://arxiv.org/abs/1904.08779).
mask_time_prob (`float`, *optional*, defaults to 0.05):
Percentage (between 0 and 1) of all feature vectors along the time axis which will be masked. The masking
procecure generates ''mask_time_prob*len(time_axis)/mask_time_length'' independent masks over the axis. If
reasoning from the propability of each feature vector to be chosen as the start of the vector span to be
- masked, *mask_time_prob* should be `prob_vector_start*mask_time_length`. Note that overlap may decrease
- the actual percentage of masked vectors. This is only relevant if `apply_spec_augment is True`.
+ masked, *mask_time_prob* should be `prob_vector_start*mask_time_length`. Note that overlap may decrease the
+ actual percentage of masked vectors. This is only relevant if `apply_spec_augment is True`.
mask_time_length (`int`, *optional*, defaults to 10):
Length of vector span along the time axis.
mask_time_min_masks (`int`, *optional*, defaults to 2),:
- The minimum number of masks of length `mask_feature_length` generated along the time axis, each time
- step, irrespectively of `mask_feature_prob`. Only relevant if
- ''mask_time_prob*len(time_axis)/mask_time_length < mask_time_min_masks''
+ The minimum number of masks of length `mask_feature_length` generated along the time axis, each time step,
+ irrespectively of `mask_feature_prob`. Only relevant if ''mask_time_prob*len(time_axis)/mask_time_length <
+ mask_time_min_masks''
mask_feature_prob (`float`, *optional*, defaults to 0.0):
Percentage (between 0 and 1) of all feature vectors along the feature axis which will be masked. The
masking procecure generates ''mask_feature_prob*len(feature_axis)/mask_time_length'' independent masks over
the axis. If reasoning from the propability of each feature vector to be chosen as the start of the vector
- span to be masked, *mask_feature_prob* should be `prob_vector_start*mask_feature_length`. Note that
- overlap may decrease the actual percentage of masked vectors. This is only relevant if `apply_spec_augment is True`.
+ span to be masked, *mask_feature_prob* should be `prob_vector_start*mask_feature_length`. Note that overlap
+ may decrease the actual percentage of masked vectors. This is only relevant if `apply_spec_augment is
+ True`.
mask_feature_length (`int`, *optional*, defaults to 10):
Length of vector span along the feature axis.
mask_feature_min_masks (`int`, *optional*, defaults to 0),:
@@ -141,9 +145,9 @@ class Wav2Vec2Config(PretrainedConfig):
Specifies the reduction to apply to the output of `torch.nn.CTCLoss`. Only relevant when training an
instance of [`Wav2Vec2ForCTC`].
ctc_zero_infinity (`bool`, *optional*, defaults to `False`):
- Whether to zero infinite losses and the associated gradients of `torch.nn.CTCLoss`. Infinite losses
- mainly occur when the inputs are too short to be aligned to the targets. Only relevant when training an
- instance of [`Wav2Vec2ForCTC`].
+ Whether to zero infinite losses and the associated gradients of `torch.nn.CTCLoss`. Infinite losses mainly
+ occur when the inputs are too short to be aligned to the targets. Only relevant when training an instance
+ of [`Wav2Vec2ForCTC`].
use_weighted_layer_sum (`bool`, *optional*, defaults to `False`):
Whether to use a weighted average of layer outputs with learned weights. Only relevant when using an
instance of [`Wav2Vec2ForSequenceClassification`].
@@ -168,7 +172,8 @@ class Wav2Vec2Config(PretrainedConfig):
adapter_stride (`int`, *optional*, defaults to 2):
Stride of the convolutional layers in the adapter network. Only relevant if `add_adapter is True`.
num_adapter_layers (`int`, *optional*, defaults to 3):
- Number of convolutional layers that should be used in the adapter network. Only relevant if `add_adapter is True`.
+ Number of convolutional layers that should be used in the adapter network. Only relevant if `add_adapter is
+ True`.
output_hidden_size (`int`, *optional*):
Dimensionality of the encoder output layer. If not defined, this defaults to *hidden-size*. Only relevant
if `add_adapter is True`.
diff --git a/src/transformers/models/wav2vec2/feature_extraction_wav2vec2.py b/src/transformers/models/wav2vec2/feature_extraction_wav2vec2.py
index 1b4894430d..3909d3ab7c 100644
--- a/src/transformers/models/wav2vec2/feature_extraction_wav2vec2.py
+++ b/src/transformers/models/wav2vec2/feature_extraction_wav2vec2.py
@@ -33,9 +33,8 @@ class Wav2Vec2FeatureExtractor(SequenceFeatureExtractor):
r"""
Constructs a Wav2Vec2 feature extractor.
- This feature extractor inherits from
- [`~feature_extraction_sequence_utils.SequenceFeatureExtractor`] which contains most of the main
- methods. Users should refer to this superclass for more information regarding those methods.
+ This feature extractor inherits from [`~feature_extraction_sequence_utils.SequenceFeatureExtractor`] which contains
+ most of the main methods. Users should refer to this superclass for more information regarding those methods.
Args:
feature_size (`int`, defaults to 1):
@@ -46,17 +45,20 @@ class Wav2Vec2FeatureExtractor(SequenceFeatureExtractor):
The value that is used to fill the padding values.
do_normalize (`bool`, *optional*, defaults to `False`):
Whether or not to zero-mean unit-variance normalize the input. Normalizing can help to significantly
- improve the performance for some models, *e.g.*, [wav2vec2-lv60](https://huggingface.co/models?search=lv60).
+ improve the performance for some models, *e.g.*,
+ [wav2vec2-lv60](https://huggingface.co/models?search=lv60).
return_attention_mask (`bool`, *optional*, defaults to `False`):
Whether or not [`~Wav2Vec2FeatureExtractor.__call__`] should return `attention_mask`.
- Wav2Vec2 models that have set `config.feat_extract_norm == "group"`, such as [wav2vec2-base](https://huggingface.co/facebook/wav2vec2-base-960h), have **not** been trained using
- `attention_mask`. For such models, `input_values` should simply be padded with 0 and no
- `attention_mask` should be passed.
+ Wav2Vec2 models that have set `config.feat_extract_norm == "group"`, such as
+ [wav2vec2-base](https://huggingface.co/facebook/wav2vec2-base-960h), have **not** been trained using
+ `attention_mask`. For such models, `input_values` should simply be padded with 0 and no `attention_mask`
+ should be passed.
- For Wav2Vec2 models that have set `config.feat_extract_norm == "layer"`, such as [wav2vec2-lv60](https://huggingface.co/facebook/wav2vec2-large-960h-lv60-self), `attention_mask` should be
+ For Wav2Vec2 models that have set `config.feat_extract_norm == "layer"`, such as
+ [wav2vec2-lv60](https://huggingface.co/facebook/wav2vec2-large-960h-lv60-self), `attention_mask` should be
passed for batched inference.
"""
@@ -121,12 +123,12 @@ class Wav2Vec2FeatureExtractor(SequenceFeatureExtractor):
Select a strategy to pad the returned sequences (according to the model's padding side and padding
index) among:
- - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a
- single sequence if provided).
- - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the
- maximum acceptable input length for the model if that argument is not provided.
- - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of
- different lengths).
+ - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
+ sequence if provided).
+ - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
+ acceptable input length for the model if that argument is not provided.
+ - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different
+ lengths).
max_length (`int`, *optional*):
Maximum length of the returned list and optionally padding length (see above).
truncation (`bool`):
@@ -144,12 +146,14 @@ class Wav2Vec2FeatureExtractor(SequenceFeatureExtractor):
- Wav2Vec2 models that have set `config.feat_extract_norm == "group"`, such as [wav2vec2-base](https://huggingface.co/facebook/wav2vec2-base-960h), have **not** been trained using
+ Wav2Vec2 models that have set `config.feat_extract_norm == "group"`, such as
+ [wav2vec2-base](https://huggingface.co/facebook/wav2vec2-base-960h), have **not** been trained using
`attention_mask`. For such models, `input_values` should simply be padded with 0 and no
`attention_mask` should be passed.
- For Wav2Vec2 models that have set `config.feat_extract_norm == "layer"`, such as [wav2vec2-lv60](https://huggingface.co/facebook/wav2vec2-large-960h-lv60-self), `attention_mask` should be
- passed for batched inference.
+ For Wav2Vec2 models that have set `config.feat_extract_norm == "layer"`, such as
+ [wav2vec2-lv60](https://huggingface.co/facebook/wav2vec2-large-960h-lv60-self), `attention_mask` should
+ be passed for batched inference.
diff --git a/src/transformers/models/wav2vec2/modeling_flax_wav2vec2.py b/src/transformers/models/wav2vec2/modeling_flax_wav2vec2.py
index 8344d43352..5764fed75d 100644
--- a/src/transformers/models/wav2vec2/modeling_flax_wav2vec2.py
+++ b/src/transformers/models/wav2vec2/modeling_flax_wav2vec2.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" Flax Wav2Vec2 model. """
+""" Flax Wav2Vec2 model."""
from functools import partial
from typing import Optional, Tuple, Union
@@ -54,12 +54,13 @@ class FlaxWav2Vec2BaseModelOutput(ModelOutput):
Sequence of extracted feature vectors of the last convolutional layer of the model with `last_conv_dim`
being the dimension of the last convolutional layer.
hidden_states (`tuple(jnp.ndarray)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(jnp.ndarray)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -74,8 +75,7 @@ class FlaxWav2Vec2BaseModelOutput(ModelOutput):
@flax.struct.dataclass
class FlaxWav2Vec2ForPreTrainingOutput(ModelOutput):
"""
- Output type of [`FlaxWav2Vec2ForPreTrainingOutput`], with potential hidden states and
- attentions.
+ Output type of [`FlaxWav2Vec2ForPreTrainingOutput`], with potential hidden states and attentions.
Args:
loss (*optional*, returned when model is in train mode, `jnp.ndarray` of shape `(1,)`):
@@ -88,12 +88,13 @@ class FlaxWav2Vec2ForPreTrainingOutput(ModelOutput):
Quantized extracted feature vectors projected to *config.proj_codevector_dim* representing the positive
target vectors for contrastive loss.
hidden_states (`tuple(jnp.ndarray)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `jnp.ndarray` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(jnp.ndarray)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `jnp.ndarray` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -115,13 +116,14 @@ def _compute_mask_indices(
) -> np.ndarray:
"""
Computes random mask spans for a given shape. Used to implement [SpecAugment: A Simple Data Augmentation Method for
- ASR](https://arxiv.org/abs/1904.08779). Note that this method is not optimized to run on TPU and should be run
- on CPU as part of the preprocessing during training.
+ ASR](https://arxiv.org/abs/1904.08779). Note that this method is not optimized to run on TPU and should be run on
+ CPU as part of the preprocessing during training.
Args:
shape: the the shape for which to compute masks.
should be of size 2 where first element is batch size and 2nd is timesteps
- mask_prob: probability for each token to be chosen as start of the span to be masked. this will be multiplied by
+ mask_prob:
+ probability for each token to be chosen as start of the span to be masked. this will be multiplied by
number of timesteps divided by length of mask span to mask approximately this percentage of all elements.
however due to overlaps, the actual number will be smaller (unless no_overlap is True)
mask_length: size of the mask
@@ -211,14 +213,17 @@ def _sample_negative_indices(features_shape: Tuple, num_negatives: int, attentio
WAV_2_VEC_2_START_DOCSTRING = r"""
- Wav2Vec2 was proposed in [wav2vec 2.0: A Framework for Self-Supervised Learning of Speech Representations](https://arxiv.org/abs/2006.11477) by Alexei Baevski, Henry Zhou, Abdelrahman Mohamed, Michael Auli.
+ Wav2Vec2 was proposed in [wav2vec 2.0: A Framework for Self-Supervised Learning of Speech
+ Representations](https://arxiv.org/abs/2006.11477) by Alexei Baevski, Henry Zhou, Abdelrahman Mohamed, Michael
+ Auli.
- This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading or saving, resizing the input
- embeddings, pruning heads etc.)
+ This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a Flax Linen [flax.nn.Module](https://flax.readthedocs.io/en/latest/_autosummary/flax.nn.module.html) subclass. Use it as a regular Flax
- Module and refer to the Flax documentation for all matter related to general usage and behavior.
+ This model is also a Flax Linen
+ [flax.nn.Module](https://flax.readthedocs.io/en/latest/_autosummary/flax.nn.module.html) subclass. Use it as a
+ regular Flax Module and refer to the Flax documentation for all matter related to general usage and behavior.
Finally, this model supports inherent JAX features such as:
@@ -230,11 +235,10 @@ WAV_2_VEC_2_START_DOCSTRING = r"""
Parameters:
config ([`Wav2Vec2Config`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the
- model weights.
+ configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the model weights.
dtype (`jax.numpy.dtype`, *optional*, defaults to `jax.numpy.float32`):
- The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on
- GPUs) and `jax.numpy.bfloat16` (on TPUs).
+ The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on GPUs) and
+ `jax.numpy.bfloat16` (on TPUs).
This can be used to enable mixed-precision training or half-precision inference on GPUs or TPUs. If
specified all the computation will be performed with the given `dtype`.
@@ -242,8 +246,8 @@ WAV_2_VEC_2_START_DOCSTRING = r"""
**Note that this only specifies the dtype of the computation and does not influence the dtype of model
parameters.**
- If you wish to change the dtype of the model parameters, see
- [`~FlaxPreTrainedModel.to_fp16`] and [`~FlaxPreTrainedModel.to_bf16`].
+ If you wish to change the dtype of the model parameters, see [`~FlaxPreTrainedModel.to_fp16`] and
+ [`~FlaxPreTrainedModel.to_bf16`].
"""
@@ -252,20 +256,21 @@ WAV_2_VEC_2_INPUTS_DOCSTRING = r"""
input_values (`jnp.ndarray` of shape `(batch_size, sequence_length)`):
Float values of input raw speech waveform. Values can be obtained by loading a *.flac* or *.wav* audio file
into an array of type *List[float]* or a *numpy.ndarray*, *e.g.* via the soundfile library (*pip install
- soundfile*). To prepare the array into *input_values*, the [`Wav2Vec2Processor`] should
- be used for padding and conversion into a tensor of type *jnp.ndarray*. See
- [`Wav2Vec2Processor.__call__`] for details.
+ soundfile*). To prepare the array into *input_values*, the [`Wav2Vec2Processor`] should be used for padding
+ and conversion into a tensor of type *jnp.ndarray*. See [`Wav2Vec2Processor.__call__`] for details.
attention_mask (`jnp.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
- Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0, 1]`:
+ Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0,
+ 1]`:
- 1 for tokens that are **not masked**,
- 0 for tokens that are **masked**.
- [What are attention masks?](../glossary#attention-mask) .. warning:: `attention_mask` should
- only be passed if the corresponding processor has `config.return_attention_mask == True`. For all models
- whose processor has `config.return_attention_mask == False`, such as [wav2vec2-base](https://huggingface.co/facebook/wav2vec2-base-960h), `attention_mask` should **not** be passed to
- avoid degraded performance when doing batched inference. For such models `input_values` should simply
- be padded with 0 and passed without `attention_mask`. Be aware that these models also yield slightly
+ [What are attention masks?](../glossary#attention-mask) .. warning:: `attention_mask` should only be passed
+ if the corresponding processor has `config.return_attention_mask == True`. For all models whose processor
+ has `config.return_attention_mask == False`, such as
+ [wav2vec2-base](https://huggingface.co/facebook/wav2vec2-base-960h), `attention_mask` should **not** be
+ passed to avoid degraded performance when doing batched inference. For such models `input_values` should
+ simply be padded with 0 and passed without `attention_mask`. Be aware that these models also yield slightly
different results depending on whether `input_values` is padded or not.
mask_time_indices (`jnp.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
Indices to mask extracted features for contrastive loss. When in training mode, model learns to predict
@@ -1167,7 +1172,7 @@ class FlaxWav2Vec2ForPreTrainingModule(nn.Module):
return input_lengths
-@add_start_docstrings("""Wav2Vec2 Model with a quantizer and `VQ` head on top. """, WAV_2_VEC_2_START_DOCSTRING)
+@add_start_docstrings("""Wav2Vec2 Model with a quantizer and `VQ` head on top.""", WAV_2_VEC_2_START_DOCSTRING)
class FlaxWav2Vec2ForPreTraining(FlaxWav2Vec2PreTrainedModel):
module_class = FlaxWav2Vec2ForPreTrainingModule
diff --git a/src/transformers/models/wav2vec2/modeling_tf_wav2vec2.py b/src/transformers/models/wav2vec2/modeling_tf_wav2vec2.py
index 1770667414..eeccb467a4 100644
--- a/src/transformers/models/wav2vec2/modeling_tf_wav2vec2.py
+++ b/src/transformers/models/wav2vec2/modeling_tf_wav2vec2.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" TensorFlow Wav2Vec2 model. """
+""" TensorFlow Wav2Vec2 model."""
import inspect
import warnings
@@ -61,7 +61,8 @@ LARGE_NEGATIVE = -1e8
def input_values_processing(func, config, input_values, **kwargs):
"""
Process the input of each TensorFlow model including the booleans. In case of a list of symbolic inputs, each input
- has to be named accordingly to the parameters name, i.e. `input_values = tf.keras.Input(shape=(128,), dtype='float32', name="input_values")` otherwise the order of the tensors will not be guaranteed during the
+ has to be named accordingly to the parameters name, i.e. `input_values = tf.keras.Input(shape=(128,),
+ dtype='float32', name="input_values")` otherwise the order of the tensors will not be guaranteed during the
training.
Args:
@@ -208,7 +209,8 @@ def _compute_mask_indices(
shape: the the shape for which to compute masks.
should be of size 2 where first element is batch size and 2nd is timesteps
attention_mask: optional padding mask of the same size as shape, which will prevent masking padded elements
- mask_prob: probability for each token to be chosen as start of the span to be masked. this will be multiplied by
+ mask_prob:
+ probability for each token to be chosen as start of the span to be masked. this will be multiplied by
number of timesteps divided by length of mask span to mask approximately this percentage of all elements.
however due to overlaps, the actual number will be smaller (unless no_overlap is True)
mask_length: size of the mask
@@ -1276,13 +1278,13 @@ class TFWav2Vec2PreTrainedModel(TFPreTrainedModel):
WAV_2_VEC_2_START_DOCSTRING = r"""
- This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading or saving, resizing the input
- embeddings, pruning heads etc.)
+ This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use
- it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage
- and behavior.
+ This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it
+ as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and
+ behavior.
@@ -1291,11 +1293,11 @@ WAV_2_VEC_2_START_DOCSTRING = r"""
- having all inputs as keyword arguments (like PyTorch models), or
- having all inputs as a list, tuple or dict in the first positional arguments.
- This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all
- the tensors in the first argument of the model call function: `model(inputs)`.
+ This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all the
+ tensors in the first argument of the model call function: `model(inputs)`.
- If you choose this second option, there are three possibilities you can use to gather all the input Tensors in
- the first positional argument :
+ If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the
+ first positional argument :
- a single Tensor with `input_values` only and nothing else: `model(inputs_ids)`
- a list of varying length with one or several input Tensors IN THE ORDER given in the docstring:
@@ -1308,8 +1310,7 @@ WAV_2_VEC_2_START_DOCSTRING = r"""
Args:
config ([`Wav2Vec2Config`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
WAV_2_VEC_2_INPUTS_DOCSTRING = r"""
@@ -1317,9 +1318,8 @@ WAV_2_VEC_2_INPUTS_DOCSTRING = r"""
input_values (`np.ndarray`, `tf.Tensor`, `List[tf.Tensor]` ``Dict[str, tf.Tensor]` or `Dict[str, np.ndarray]` and each example must have the shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`BertTokenizer`]. See
- [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for
- details.
+ Indices can be obtained using [`BertTokenizer`]. See [`PreTrainedTokenizer.__call__`] and
+ [`PreTrainedTokenizer.encode`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`np.ndarray` or `tf.Tensor` of shape `({0})`, *optional*):
@@ -1330,14 +1330,16 @@ WAV_2_VEC_2_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
token_type_ids (`np.ndarray` or `tf.Tensor` of shape `({0})`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`np.ndarray` or `tf.Tensor` of shape `({0})`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
head_mask (`np.ndarray` or `tf.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
@@ -1347,9 +1349,9 @@ WAV_2_VEC_2_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
inputs_embeds (`np.ndarray` or `tf.Tensor` of shape `({0}, hidden_size)`, *optional*):
- Optionally, instead of passing `input_values` you can choose to directly pass an embedded
- representation. This is useful if you want more control over how to convert `input_values` indices
- into associated vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_values` you can choose to directly pass an embedded representation.
+ This is useful if you want more control over how to convert `input_values` indices into associated vectors
+ than the model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the
@@ -1359,8 +1361,8 @@ WAV_2_VEC_2_INPUTS_DOCSTRING = r"""
more detail. This argument can be used only in eager mode, in graph mode the value in the config will be
used instead.
return_dict (`bool`, *optional*):
- Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This
- argument can be used in eager mode, in graph mode the value will always be set to True.
+ Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This argument can be used
+ in eager mode, in graph mode the value will always be set to True.
training (`bool`, *optional*, defaults to `False``):
Whether or not to use the model in training mode (some modules like dropout modules have different
behaviors between training and evaluation).
@@ -1464,7 +1466,7 @@ class TFWav2Vec2Model(TFWav2Vec2PreTrainedModel):
@add_start_docstrings(
- """TFWav2Vec2 Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC). """,
+ """TFWav2Vec2 Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC).""",
WAV_2_VEC_2_START_DOCSTRING,
)
class TFWav2Vec2ForCTC(TFWav2Vec2PreTrainedModel):
@@ -1500,8 +1502,9 @@ class TFWav2Vec2ForCTC(TFWav2Vec2PreTrainedModel):
) -> Union[TFCausalLMOutput, Tuple[tf.Tensor]]:
r"""
labels (`tf.Tensor` or `np.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_values` docstring) Tokens with indices set to `-100` are ignored
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
+ Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
+ config.vocab_size]` (see `input_values` docstring) Tokens with indices set to `-100` are ignored (masked),
+ the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
Returns:
diff --git a/src/transformers/models/wav2vec2/modeling_wav2vec2.py b/src/transformers/models/wav2vec2/modeling_wav2vec2.py
index 9fcb1b52b2..9d28d4980a 100755
--- a/src/transformers/models/wav2vec2/modeling_wav2vec2.py
+++ b/src/transformers/models/wav2vec2/modeling_wav2vec2.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" PyTorch Wav2Vec2 model. """
+""" PyTorch Wav2Vec2 model."""
import math
import warnings
@@ -80,12 +80,13 @@ class Wav2Vec2BaseModelOutput(ModelOutput):
extract_features (`torch.FloatTensor` of shape `(batch_size, sequence_length, conv_dim[-1])`):
Sequence of extracted feature vectors of the last convolutional layer of the model.
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -113,12 +114,13 @@ class Wav2Vec2ForPreTrainingOutput(ModelOutput):
Quantized extracted feature vectors projected to *config.proj_codevector_dim* representing the positive
target vectors for contrastive loss.
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -151,12 +153,13 @@ class XVectorOutput(ModelOutput):
embeddings (`torch.FloatTensor` of shape `(batch_size, config.xvector_output_dim)`):
Utterance embeddings used for vector similarity-based retrieval.
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -178,8 +181,8 @@ def _compute_mask_indices(
) -> np.ndarray:
"""
Computes random mask spans for a given shape. Used to implement [SpecAugment: A Simple Data Augmentation Method for
- ASR](https://arxiv.org/abs/1904.08779). Note that this method is not optimized to run on TPU and should be run
- on CPU as part of the preprocessing during training.
+ ASR](https://arxiv.org/abs/1904.08779). Note that this method is not optimized to run on TPU and should be run on
+ CPU as part of the preprocessing during training.
Args:
shape: The shape for which to compute masks. This should be of a tuple of size 2 where
@@ -1127,10 +1130,12 @@ class Wav2Vec2PreTrainedModel(PreTrainedModel):
WAV_2_VEC_2_START_DOCSTRING = r"""
- Wav2Vec2 was proposed in [wav2vec 2.0: A Framework for Self-Supervised Learning of Speech Representations](https://arxiv.org/abs/2006.11477) by Alexei Baevski, Henry Zhou, Abdelrahman Mohamed, Michael Auli.
+ Wav2Vec2 was proposed in [wav2vec 2.0: A Framework for Self-Supervised Learning of Speech
+ Representations](https://arxiv.org/abs/2006.11477) by Alexei Baevski, Henry Zhou, Abdelrahman Mohamed, Michael
+ Auli.
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving etc.).
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving etc.).
This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use
it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
@@ -1139,8 +1144,7 @@ WAV_2_VEC_2_START_DOCSTRING = r"""
Parameters:
config ([`Wav2Vec2Config`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
@@ -1149,11 +1153,11 @@ WAV_2_VEC_2_INPUTS_DOCSTRING = r"""
input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`):
Float values of input raw speech waveform. Values can be obtained by loading a *.flac* or *.wav* audio file
into an array of type *List[float]* or a *numpy.ndarray*, *e.g.* via the soundfile library (*pip install
- soundfile*). To prepare the array into *input_values*, the [`Wav2Vec2Processor`] should
- be used for padding and conversion into a tensor of type *torch.FloatTensor*. See
- [`Wav2Vec2Processor.__call__`] for details.
+ soundfile*). To prepare the array into *input_values*, the [`Wav2Vec2Processor`] should be used for padding
+ and conversion into a tensor of type *torch.FloatTensor*. See [`Wav2Vec2Processor.__call__`] for details.
attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0, 1]`:
+ Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0,
+ 1]`:
- 1 for tokens that are **not masked**,
- 0 for tokens that are **masked**.
@@ -1162,12 +1166,12 @@ WAV_2_VEC_2_INPUTS_DOCSTRING = r"""
- `attention_mask` should only be passed if the corresponding processor has
- `config.return_attention_mask == True`. For all models whose processor has
- `config.return_attention_mask == False`, such as [wav2vec2-base](https://huggingface.co/facebook/wav2vec2-base-960h), `attention_mask` should **not** be passed
- to avoid degraded performance when doing batched inference. For such models `input_values` should
- simply be padded with 0 and passed without `attention_mask`. Be aware that these models also yield
- slightly different results depending on whether `input_values` is padded or not.
+ `attention_mask` should only be passed if the corresponding processor has `config.return_attention_mask ==
+ True`. For all models whose processor has `config.return_attention_mask == False`, such as
+ [wav2vec2-base](https://huggingface.co/facebook/wav2vec2-base-960h), `attention_mask` should **not** be
+ passed to avoid degraded performance when doing batched inference. For such models `input_values` should
+ simply be padded with 0 and passed without `attention_mask`. Be aware that these models also yield slightly
+ different results depending on whether `input_values` is padded or not.
@@ -1312,7 +1316,7 @@ class Wav2Vec2Model(Wav2Vec2PreTrainedModel):
)
-@add_start_docstrings("""Wav2Vec2 Model with a quantizer and `VQ` head on top. """, WAV_2_VEC_2_START_DOCSTRING)
+@add_start_docstrings("""Wav2Vec2 Model with a quantizer and `VQ` head on top.""", WAV_2_VEC_2_START_DOCSTRING)
class Wav2Vec2ForPreTraining(Wav2Vec2PreTrainedModel):
def __init__(self, config: Wav2Vec2Config):
super().__init__(config)
@@ -1521,7 +1525,7 @@ class Wav2Vec2ForPreTraining(Wav2Vec2PreTrainedModel):
)
-@add_start_docstrings("""Wav2Vec2 Model with a `language modeling` head on top. """, WAV_2_VEC_2_START_DOCSTRING)
+@add_start_docstrings("""Wav2Vec2 Model with a `language modeling` head on top.""", WAV_2_VEC_2_START_DOCSTRING)
class Wav2Vec2ForMaskedLM(Wav2Vec2PreTrainedModel):
def __init__(self, config):
super().__init__(config)
@@ -1600,7 +1604,7 @@ class Wav2Vec2ForMaskedLM(Wav2Vec2PreTrainedModel):
@add_start_docstrings(
- """Wav2Vec2 Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC). """,
+ """Wav2Vec2 Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC).""",
WAV_2_VEC_2_START_DOCSTRING,
)
class Wav2Vec2ForCTC(Wav2Vec2PreTrainedModel):
@@ -1648,7 +1652,9 @@ class Wav2Vec2ForCTC(Wav2Vec2PreTrainedModel):
r"""
labels (`torch.LongTensor` of shape `(batch_size, target_length)`, *optional*):
Labels for connectionist temporal classification. Note that `target_length` has to be smaller or equal to
- the sequence length of the output logits. Indices are selected in `[-100, 0, ..., config.vocab_size - 1]`. All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size - 1]`.
+ the sequence length of the output logits. Indices are selected in `[-100, 0, ..., config.vocab_size - 1]`.
+ All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ...,
+ config.vocab_size - 1]`.
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1762,8 +1768,9 @@ class Wav2Vec2ForSequenceClassification(Wav2Vec2PreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1863,8 +1870,9 @@ class Wav2Vec2ForAudioFrameClassification(Wav2Vec2PreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -2024,8 +2032,9 @@ class Wav2Vec2ForXVector(Wav2Vec2PreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
diff --git a/src/transformers/models/wav2vec2/processing_wav2vec2.py b/src/transformers/models/wav2vec2/processing_wav2vec2.py
index 3b1313665a..4cf2a200ff 100644
--- a/src/transformers/models/wav2vec2/processing_wav2vec2.py
+++ b/src/transformers/models/wav2vec2/processing_wav2vec2.py
@@ -29,10 +29,8 @@ class Wav2Vec2Processor:
Constructs a Wav2Vec2 processor which wraps a Wav2Vec2 feature extractor and a Wav2Vec2 CTC tokenizer into a single
processor.
- [`Wav2Vec2Processor`] offers all the functionalities of
- [`Wav2Vec2FeatureExtractor`] and [`PreTrainedTokenizer`]. See the docstring
- of [`~Wav2Vec2Processor.__call__`] and [`~Wav2Vec2Processor.decode`] for more
- information.
+ [`Wav2Vec2Processor`] offers all the functionalities of [`Wav2Vec2FeatureExtractor`] and [`PreTrainedTokenizer`].
+ See the docstring of [`~Wav2Vec2Processor.__call__`] and [`~Wav2Vec2Processor.decode`] for more information.
Args:
feature_extractor (`Wav2Vec2FeatureExtractor`):
@@ -62,10 +60,9 @@ class Wav2Vec2Processor:
- This class method is simply calling
- [`~feature_extraction_utils.FeatureExtractionMixin.save_pretrained`] and
- [`~tokenization_utils_base.PreTrainedTokenizer.save_pretrained`]. Please refer to the
- docstrings of the methods above for more information.
+ This class method is simply calling [`~feature_extraction_utils.FeatureExtractionMixin.save_pretrained`] and
+ [`~tokenization_utils_base.PreTrainedTokenizer.save_pretrained`]. Please refer to the docstrings of the methods
+ above for more information.
@@ -86,9 +83,9 @@ class Wav2Vec2Processor:
This class method is simply calling Wav2Vec2FeatureExtractor's
- [`~feature_extraction_utils.FeatureExtractionMixin.from_pretrained`] and
- PreTrainedTokenizer's [`~tokenization_utils_base.PreTrainedTokenizer.from_pretrained`].
- Please refer to the docstrings of the methods above for more information.
+ [`~feature_extraction_utils.FeatureExtractionMixin.from_pretrained`] and PreTrainedTokenizer's
+ [`~tokenization_utils_base.PreTrainedTokenizer.from_pretrained`]. Please refer to the docstrings of the methods
+ above for more information.
@@ -100,8 +97,7 @@ class Wav2Vec2Processor:
huggingface.co. Valid model ids can be located at the root-level, like `bert-base-uncased`, or
namespaced under a user or organization name, like `dbmdz/bert-base-german-cased`.
- a path to a *directory* containing a feature extractor file saved using the
- [`~SequenceFeatureExtractor.save_pretrained`] method, e.g.,
- `./my_model_directory/`.
+ [`~SequenceFeatureExtractor.save_pretrained`] method, e.g., `./my_model_directory/`.
- a path or url to a saved feature extractor JSON *file*, e.g.,
`./my_model_directory/preprocessor_config.json`.
**kwargs
@@ -133,9 +129,8 @@ class Wav2Vec2Processor:
"""
When used in normal mode, this method forwards all its arguments to Wav2Vec2FeatureExtractor's
[`~Wav2Vec2FeatureExtractor.__call__`] and returns its output. If used in the context
- [`~Wav2Vec2Processor.as_target_processor`] this method forwards all its arguments to
- PreTrainedTokenizer's [`~PreTrainedTokenizer.__call__`]. Please refer to the docstring of the
- above two methods for more information.
+ [`~Wav2Vec2Processor.as_target_processor`] this method forwards all its arguments to PreTrainedTokenizer's
+ [`~PreTrainedTokenizer.__call__`]. Please refer to the docstring of the above two methods for more information.
"""
return self.current_processor(*args, **kwargs)
@@ -143,25 +138,22 @@ class Wav2Vec2Processor:
"""
When used in normal mode, this method forwards all its arguments to Wav2Vec2FeatureExtractor's
[`~Wav2Vec2FeatureExtractor.pad`] and returns its output. If used in the context
- [`~Wav2Vec2Processor.as_target_processor`] this method forwards all its arguments to
- PreTrainedTokenizer's [`~PreTrainedTokenizer.pad`]. Please refer to the docstring of the above
- two methods for more information.
+ [`~Wav2Vec2Processor.as_target_processor`] this method forwards all its arguments to PreTrainedTokenizer's
+ [`~PreTrainedTokenizer.pad`]. Please refer to the docstring of the above two methods for more information.
"""
return self.current_processor.pad(*args, **kwargs)
def batch_decode(self, *args, **kwargs):
"""
- This method forwards all its arguments to PreTrainedTokenizer's
- [`~PreTrainedTokenizer.batch_decode`]. Please refer to the docstring of this method for more
- information.
+ This method forwards all its arguments to PreTrainedTokenizer's [`~PreTrainedTokenizer.batch_decode`]. Please
+ refer to the docstring of this method for more information.
"""
return self.tokenizer.batch_decode(*args, **kwargs)
def decode(self, *args, **kwargs):
"""
- This method forwards all its arguments to PreTrainedTokenizer's
- [`~PreTrainedTokenizer.decode`]. Please refer to the docstring of this method for more
- information.
+ This method forwards all its arguments to PreTrainedTokenizer's [`~PreTrainedTokenizer.decode`]. Please refer
+ to the docstring of this method for more information.
"""
return self.tokenizer.decode(*args, **kwargs)
diff --git a/src/transformers/models/wav2vec2/tokenization_wav2vec2.py b/src/transformers/models/wav2vec2/tokenization_wav2vec2.py
index d2e024ed0b..ed9b32b0b9 100644
--- a/src/transformers/models/wav2vec2/tokenization_wav2vec2.py
+++ b/src/transformers/models/wav2vec2/tokenization_wav2vec2.py
@@ -53,18 +53,18 @@ WAV2VEC2_KWARGS_DOCSTRING = r"""
padding (`bool`, `str` or [`~file_utils.PaddingStrategy`], *optional*, defaults to `False`):
Activates and controls padding. Accepts the following values:
- - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a
- single sequence if provided).
- - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the
- maximum acceptable input length for the model if that argument is not provided.
- - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of
- different lengths).
+ - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
+ sequence if provided).
+ - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
+ acceptable input length for the model if that argument is not provided.
+ - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different
+ lengths).
max_length (`int`, *optional*):
Controls the maximum length to use by one of the truncation/padding parameters.
- If left unset or set to `None`, this will use the predefined model maximum length if a maximum
- length is required by one of the truncation/padding parameters. If the model has no specific maximum
- input length (like XLNet) truncation/padding to a maximum length will be deactivated.
+ If left unset or set to `None`, this will use the predefined model maximum length if a maximum length
+ is required by one of the truncation/padding parameters. If the model has no specific maximum input
+ length (like XLNet) truncation/padding to a maximum length will be deactivated.
pad_to_multiple_of (`int`, *optional*):
If set will pad the sequence to a multiple of the provided value. This is especially useful to enable
the use of Tensor Cores on NVIDIA hardware with compute capability >= 7.5 (Volta).
@@ -84,8 +84,8 @@ class Wav2Vec2CTCTokenizer(PreTrainedTokenizer):
"""
Constructs a Wav2Vec2CTC tokenizer.
- This tokenizer inherits from [`PreTrainedTokenizer`] which contains some of the main methods.
- Users should refer to the superclass for more information regarding such methods.
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains some of the main methods. Users should refer to
+ the superclass for more information regarding such methods.
Args:
vocab_file (`str`):
@@ -163,8 +163,8 @@ class Wav2Vec2CTCTokenizer(PreTrainedTokenizer):
@property
def word_delimiter_token_id(self) -> Optional[int]:
"""
- `Optional[int]`: Id of the word_delimiter_token in the vocabulary. Returns `None` if the token has
- not been set.
+ `Optional[int]`: Id of the word_delimiter_token in the vocabulary. Returns `None` if the token has not been
+ set.
"""
if self._word_delimiter_token is None:
return None
@@ -342,8 +342,8 @@ class Wav2Vec2Tokenizer(PreTrainedTokenizer):
"""
Constructs a Wav2Vec2 tokenizer.
- This tokenizer inherits from [`PreTrainedTokenizer`] which contains some of the main methods.
- Users should refer to the superclass for more information regarding such methods.
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains some of the main methods. Users should refer to
+ the superclass for more information regarding such methods.
Args:
vocab_file (`str`):
@@ -363,17 +363,20 @@ class Wav2Vec2Tokenizer(PreTrainedTokenizer):
Whether or not to lowercase the output when decoding.
do_normalize (`bool`, *optional*, defaults to `False`):
Whether or not to zero-mean unit-variance normalize the input. Normalizing can help to significantly
- improve the performance for some models, *e.g.*, [wav2vec2-lv60](https://huggingface.co/models?search=lv60).
+ improve the performance for some models, *e.g.*,
+ [wav2vec2-lv60](https://huggingface.co/models?search=lv60).
return_attention_mask (`bool`, *optional*, defaults to `False`):
Whether or not [`~Wav2Vec2Tokenizer.__call__`] should return `attention_mask`.
- Wav2Vec2 models that have set `config.feat_extract_norm == "group"`, such as [wav2vec2-base](https://huggingface.co/facebook/wav2vec2-base-960h), have **not** been trained using
- `attention_mask`. For such models, `input_values` should simply be padded with 0 and no
- `attention_mask` should be passed.
+ Wav2Vec2 models that have set `config.feat_extract_norm == "group"`, such as
+ [wav2vec2-base](https://huggingface.co/facebook/wav2vec2-base-960h), have **not** been trained using
+ `attention_mask`. For such models, `input_values` should simply be padded with 0 and no `attention_mask`
+ should be passed.
- For Wav2Vec2 models that have set `config.feat_extract_norm == "layer"`, such as [wav2vec2-lv60](https://huggingface.co/facebook/wav2vec2-large-960h-lv60-self), `attention_mask` should be
+ For Wav2Vec2 models that have set `config.feat_extract_norm == "layer"`, such as
+ [wav2vec2-lv60](https://huggingface.co/facebook/wav2vec2-large-960h-lv60-self), `attention_mask` should be
passed for batched inference.
@@ -447,8 +450,8 @@ class Wav2Vec2Tokenizer(PreTrainedTokenizer):
@property
def word_delimiter_token_id(self) -> Optional[int]:
"""
- `Optional[int]`: Id of the word_delimiter_token in the vocabulary. Returns `None` if the token has
- not been set.
+ `Optional[int]`: Id of the word_delimiter_token in the vocabulary. Returns `None` if the token has not been
+ set.
"""
if self._word_delimiter_token is None:
return None
diff --git a/src/transformers/models/wav2vec2_phoneme/tokenization_wav2vec2_phoneme.py b/src/transformers/models/wav2vec2_phoneme/tokenization_wav2vec2_phoneme.py
index 0fc8cb7a49..d1de690e4a 100644
--- a/src/transformers/models/wav2vec2_phoneme/tokenization_wav2vec2_phoneme.py
+++ b/src/transformers/models/wav2vec2_phoneme/tokenization_wav2vec2_phoneme.py
@@ -52,8 +52,8 @@ class Wav2Vec2PhonemeCTCTokenizer(PreTrainedTokenizer):
"""
Constructs a Wav2Vec2PhonemeCTC tokenizer.
- This tokenizer inherits from [`PreTrainedTokenizer`] which contains some of the main methods.
- Users should refer to the superclass for more information regarding such methods.
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains some of the main methods. Users should refer to
+ the superclass for more information regarding such methods.
Args:
vocab_file (`str`):
@@ -226,8 +226,8 @@ class Wav2Vec2PhonemeCTCTokenizer(PreTrainedTokenizer):
@property
def word_delimiter_token_id(self) -> Optional[int]:
"""
- `Optional[int]`: Id of the word_delimiter_token in the vocabulary. Returns `None` if the token has
- not been set.
+ `Optional[int]`: Id of the word_delimiter_token in the vocabulary. Returns `None` if the token has not been
+ set.
"""
if self._word_delimiter_token is None:
return None
@@ -254,8 +254,8 @@ class Wav2Vec2PhonemeCTCTokenizer(PreTrainedTokenizer):
@property
def phone_delimiter_token_id(self) -> Optional[int]:
"""
- `Optional[int]`: Id of the phone_delimiter_token in the vocabulary. Returns `None` if the token has
- not been set.
+ `Optional[int]`: Id of the phone_delimiter_token in the vocabulary. Returns `None` if the token has not been
+ set.
"""
if self._phone_delimiter_token is None:
return None
diff --git a/src/transformers/models/wav2vec2_with_lm/processing_wav2vec2_with_lm.py b/src/transformers/models/wav2vec2_with_lm/processing_wav2vec2_with_lm.py
index be0108173d..9b1e30be86 100644
--- a/src/transformers/models/wav2vec2_with_lm/processing_wav2vec2_with_lm.py
+++ b/src/transformers/models/wav2vec2_with_lm/processing_wav2vec2_with_lm.py
@@ -98,13 +98,12 @@ class Wav2Vec2ProcessorWithLM:
def save_pretrained(self, save_directory):
"""
Save the Wav2Vec2 feature_extractor, a tokenizer object and a pyctcdecode decoder to the directory
- `save_directory`, so that they can be re-loaded using the
- [`~Wav2Vec2ProcessorWithLM.from_pretrained`] class method.
+ `save_directory`, so that they can be re-loaded using the [`~Wav2Vec2ProcessorWithLM.from_pretrained`] class
+ method.
- This class method is simply calling
- [`~feature_extraction_utils.FeatureExtractionMixin.save_pretrained,`]
+ This class method is simply calling [`~feature_extraction_utils.FeatureExtractionMixin.save_pretrained,`]
[`~tokenization_utils_base.PreTrainedTokenizer.save_pretrained`] and pyctcdecode's
[`pyctcdecode.BeamSearchDecoderCTC.save_to_dir`].
@@ -129,9 +128,9 @@ class Wav2Vec2ProcessorWithLM:
This class method is simply calling Wav2Vec2FeatureExtractor's
- [`~feature_extraction_utils.FeatureExtractionMixin.from_pretrained`],
- Wav2Vec2CTCTokenizer's [`~tokenization_utils_base.PreTrainedTokenizer.from_pretrained`],
- and [`pyctcdecode.BeamSearchDecoderCTC.load_from_hf_hub`].
+ [`~feature_extraction_utils.FeatureExtractionMixin.from_pretrained`], Wav2Vec2CTCTokenizer's
+ [`~tokenization_utils_base.PreTrainedTokenizer.from_pretrained`], and
+ [`pyctcdecode.BeamSearchDecoderCTC.load_from_hf_hub`].
Please refer to the docstrings of the methods above for more information.
@@ -145,8 +144,7 @@ class Wav2Vec2ProcessorWithLM:
huggingface.co. Valid model ids can be located at the root-level, like `bert-base-uncased`, or
namespaced under a user or organization name, like `dbmdz/bert-base-german-cased`.
- a path to a *directory* containing a feature extractor file saved using the
- [`~SequenceFeatureExtractor.save_pretrained`] method, e.g.,
- `./my_model_directory/`.
+ [`~SequenceFeatureExtractor.save_pretrained`] method, e.g., `./my_model_directory/`.
- a path or url to a saved feature extractor JSON *file*, e.g.,
`./my_model_directory/preprocessor_config.json`.
**kwargs
@@ -221,8 +219,8 @@ class Wav2Vec2ProcessorWithLM:
When used in normal mode, this method forwards all its arguments to Wav2Vec2FeatureExtractor's
[`~Wav2Vec2FeatureExtractor.__call__`] and returns its output. If used in the context
[`~Wav2Vec2ProcessorWithLM.as_target_processor`] this method forwards all its arguments to
- Wav2Vec2CTCTokenizer's [`~Wav2Vec2CTCTokenizer.__call__`]. Please refer to the docstring of
- the above two methods for more information.
+ Wav2Vec2CTCTokenizer's [`~Wav2Vec2CTCTokenizer.__call__`]. Please refer to the docstring of the above two
+ methods for more information.
"""
return self.current_processor(*args, **kwargs)
@@ -231,8 +229,8 @@ class Wav2Vec2ProcessorWithLM:
When used in normal mode, this method forwards all its arguments to Wav2Vec2FeatureExtractor's
[`~Wav2Vec2FeatureExtractor.pad`] and returns its output. If used in the context
[`~Wav2Vec2ProcessorWithLM.as_target_processor`] this method forwards all its arguments to
- Wav2Vec2CTCTokenizer's [`~Wav2Vec2CTCTokenizer.pad`]. Please refer to the docstring of the
- above two methods for more information.
+ Wav2Vec2CTCTokenizer's [`~Wav2Vec2CTCTokenizer.pad`]. Please refer to the docstring of the above two methods
+ for more information.
"""
return self.current_processor.pad(*args, **kwargs)
diff --git a/src/transformers/models/wavlm/configuration_wavlm.py b/src/transformers/models/wavlm/configuration_wavlm.py
index c6a848506d..0590cb2c88 100644
--- a/src/transformers/models/wavlm/configuration_wavlm.py
+++ b/src/transformers/models/wavlm/configuration_wavlm.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" WavLM model configuration """
+""" WavLM model configuration"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging
@@ -28,20 +28,20 @@ WAVLM_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class WavLMConfig(PretrainedConfig):
r"""
- This is the configuration class to store the configuration of a [`WavLMModel`]. It is used to
- instantiate an WavLM model according to the specified arguments, defining the model architecture. Instantiating a
- configuration with the defaults will yield a similar configuration to that of the WavLM [facebook/wavlm-base-960h](https://huggingface.co/facebook/wavlm-base-960h) architecture.
+ This is the configuration class to store the configuration of a [`WavLMModel`]. It is used to instantiate an WavLM
+ model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
+ defaults will yield a similar configuration to that of the WavLM
+ [facebook/wavlm-base-960h](https://huggingface.co/facebook/wavlm-base-960h) architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
vocab_size (`int`, *optional*, defaults to 32):
Vocabulary size of the WavLM model. Defines the number of different tokens that can be represented by the
- `inputs_ids` passed when calling [`WavLMModel`]. Vocabulary size of the model.
- Defines the different tokens that can be represented by the *inputs_ids* passed to the forward method of
- [`WavLMModel`].
+ `inputs_ids` passed when calling [`WavLMModel`]. Vocabulary size of the model. Defines the different tokens
+ that can be represented by the *inputs_ids* passed to the forward method of [`WavLMModel`].
hidden_size (`int`, *optional*, defaults to 768):
Dimensionality of the encoder layers and the pooler layer.
num_hidden_layers (`int`, *optional*, defaults to 12):
@@ -51,8 +51,8 @@ class WavLMConfig(PretrainedConfig):
intermediate_size (`int`, *optional*, defaults to 3072):
Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`):
- The non-linear activation function (function or string) in the encoder and pooler. If string,
- `"gelu"`, `"relu"`, `"selu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+ `"relu"`, `"selu"` and `"gelu_new"` are supported.
hidden_dropout (`float`, *optional*, defaults to 0.1):
The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
attention_dropout (`float`, *optional*, defaults to 0.1):
@@ -92,24 +92,27 @@ class WavLMConfig(PretrainedConfig):
num_conv_pos_embedding_groups (`int`, *optional*, defaults to 16):
Number of groups of 1D convolutional positional embeddings layer.
do_stable_layer_norm (`bool`, *optional*, defaults to `False`):
- Whether to apply *stable* layer norm architecture of the Transformer encoder. `do_stable_layer_norm is True` corresponds to applying layer norm before the attention layer, whereas `do_stable_layer_norm is False` corresponds to applying layer norm after the attention layer.
+ Whether to apply *stable* layer norm architecture of the Transformer encoder. `do_stable_layer_norm is
+ True` corresponds to applying layer norm before the attention layer, whereas `do_stable_layer_norm is
+ False` corresponds to applying layer norm after the attention layer.
apply_spec_augment (`bool`, *optional*, defaults to `True`):
Whether to apply *SpecAugment* data augmentation to the outputs of the feature extractor. For reference see
- [SpecAugment: A Simple Data Augmentation Method for Automatic Speech Recognition](https://arxiv.org/abs/1904.08779).
+ [SpecAugment: A Simple Data Augmentation Method for Automatic Speech
+ Recognition](https://arxiv.org/abs/1904.08779).
mask_time_prob (`float`, *optional*, defaults to 0.05):
Propability of each feature vector along the time axis to be chosen as the start of the vector span to be
- masked. Approximately `mask_time_prob * sequence_length // mask_time_length` feature vectors will be
- masked along the time axis. This is only relevant if `apply_spec_augment is True`.
+ masked. Approximately `mask_time_prob * sequence_length // mask_time_length` feature vectors will be masked
+ along the time axis. This is only relevant if `apply_spec_augment is True`.
mask_time_length (`int`, *optional*, defaults to 10):
Length of vector span along the time axis.
mask_time_min_masks (`int`, *optional*, defaults to 2),:
- The minimum number of masks of length `mask_feature_length` generated along the time axis, each time
- step, irrespectively of `mask_feature_prob`. Only relevant if
- ''mask_time_prob*len(time_axis)/mask_time_length < mask_time_min_masks''
+ The minimum number of masks of length `mask_feature_length` generated along the time axis, each time step,
+ irrespectively of `mask_feature_prob`. Only relevant if ''mask_time_prob*len(time_axis)/mask_time_length <
+ mask_time_min_masks''
mask_feature_prob (`float`, *optional*, defaults to 0.0):
Propability of each feature vector along the feature axis to be chosen as the start of the vector span to
- be masked. Approximately `mask_time_prob * hidden_size // mask_time_length` feature vectors will be
- masked along the time axis. This is only relevant if `apply_spec_augment is True`.
+ be masked. Approximately `mask_time_prob * hidden_size // mask_time_length` feature vectors will be masked
+ along the time axis. This is only relevant if `apply_spec_augment is True`.
mask_feature_length (`int`, *optional*, defaults to 10):
Length of vector span along the feature axis.
num_codevectors_per_group (`int`, *optional*, defaults to 320):
@@ -132,9 +135,9 @@ class WavLMConfig(PretrainedConfig):
Specifies the reduction to apply to the output of `torch.nn.CTCLoss`. Only relevant when training an
instance of [`WavLMForCTC`].
ctc_zero_infinity (`bool`, *optional*, defaults to `False`):
- Whether to zero infinite losses and the associated gradients of `torch.nn.CTCLoss`. Infinite losses
- mainly occur when the inputs are too short to be aligned to the targets. Only relevant when training an
- instance of [`WavLMForCTC`].
+ Whether to zero infinite losses and the associated gradients of `torch.nn.CTCLoss`. Infinite losses mainly
+ occur when the inputs are too short to be aligned to the targets. Only relevant when training an instance
+ of [`WavLMForCTC`].
use_weighted_layer_sum (`bool`, *optional*, defaults to `False`):
Whether to use a weighted average of layer outputs with learned weights. Only relevant when using an
instance of [`WavLMForSequenceClassification`].
@@ -159,7 +162,8 @@ class WavLMConfig(PretrainedConfig):
adapter_stride (`int`, *optional*, defaults to 2):
Stride of the convolutional layers in the adapter network. Only relevant if `add_adapter is True`.
num_adapter_layers (`int`, *optional*, defaults to 3):
- Number of convolutional layers that should be used in the adapter network. Only relevant if `add_adapter is True`.
+ Number of convolutional layers that should be used in the adapter network. Only relevant if `add_adapter is
+ True`.
output_hidden_size (`int`, *optional*):
Dimensionality of the encoder output layer. If not defined, this defaults to *hidden-size*. Only relevant
if `add_adapter is True`.
diff --git a/src/transformers/models/wavlm/modeling_wavlm.py b/src/transformers/models/wavlm/modeling_wavlm.py
index a6e82430b4..3b33b51d94 100755
--- a/src/transformers/models/wavlm/modeling_wavlm.py
+++ b/src/transformers/models/wavlm/modeling_wavlm.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" PyTorch WavLM model. """
+""" PyTorch WavLM model."""
import math
from dataclasses import dataclass
@@ -73,12 +73,13 @@ class WavLMBaseModelOutput(ModelOutput):
extract_features (`torch.FloatTensor` of shape `(batch_size, sequence_length, conv_dim[-1])`):
Sequence of extracted feature vectors of the last convolutional layer of the model.
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -103,12 +104,13 @@ class XVectorOutput(ModelOutput):
embeddings (`torch.FloatTensor` of shape `(batch_size, config.xvector_output_dim)`):
Utterance embeddings used for vector similarity-based retrieval.
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -131,8 +133,8 @@ def _compute_mask_indices(
) -> np.ndarray:
"""
Computes random mask spans for a given shape. Used to implement [SpecAugment: A Simple Data Augmentation Method for
- ASR](https://arxiv.org/abs/1904.08779). Note that this method is not optimized to run on TPU and should be run
- on CPU as part of the preprocessing during training.
+ ASR](https://arxiv.org/abs/1904.08779). Note that this method is not optimized to run on TPU and should be run on
+ CPU as part of the preprocessing during training.
Args:
shape: The shape for which to compute masks. This should be of a tuple of size 2 where
@@ -1080,11 +1082,12 @@ class WavLMPreTrainedModel(PreTrainedModel):
WAVLM_START_DOCSTRING = r"""
- WavLM was proposed in [WavLM: Unified Speech Representation Learning with Labeled and Unlabeled Data](https://arxiv.org/abs/2101.07597) by Chengyi Wang, Yu Wu, Yao Qian, Kenichi Kumatani, Shujie Liu, Furu Wei,
+ WavLM was proposed in [WavLM: Unified Speech Representation Learning with Labeled and Unlabeled
+ Data](https://arxiv.org/abs/2101.07597) by Chengyi Wang, Yu Wu, Yao Qian, Kenichi Kumatani, Shujie Liu, Furu Wei,
Michael Zeng, Xuedong Huang.
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving etc.).
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving etc.).
This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use
it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
@@ -1093,8 +1096,7 @@ WAVLM_START_DOCSTRING = r"""
Parameters:
config ([`WavLMConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
@@ -1103,11 +1105,11 @@ WAVLM_INPUTS_DOCSTRING = r"""
input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`):
Float values of input raw speech waveform. Values can be obtained by loading a *.flac* or *.wav* audio file
into an array of type *List[float]* or a *numpy.ndarray*, *e.g.* via the soundfile library (*pip install
- soundfile*). To prepare the array into *input_values*, the [`WavLMProcessor`] should be
- used for padding and conversion into a tensor of type *torch.FloatTensor*. See
- [`WavLMProcessor.__call__`] for details.
+ soundfile*). To prepare the array into *input_values*, the [`WavLMProcessor`] should be used for padding
+ and conversion into a tensor of type *torch.FloatTensor*. See [`WavLMProcessor.__call__`] for details.
attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
- Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0, 1]`:
+ Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0,
+ 1]`:
- 1 for tokens that are **not masked**,
- 0 for tokens that are **masked**.
@@ -1116,12 +1118,11 @@ WAVLM_INPUTS_DOCSTRING = r"""
- `attention_mask` should only be passed if the corresponding processor has
- `config.return_attention_mask == True`. For all models whose processor has
- `config.return_attention_mask == False`, `attention_mask` should **not** be passed to avoid
- degraded performance when doing batched inference. For such models `input_values` should simply be
- padded with 0 and passed without `attention_mask`. Be aware that these models also yield slightly
- different results depending on whether `input_values` is padded or not.
+ `attention_mask` should only be passed if the corresponding processor has `config.return_attention_mask ==
+ True`. For all models whose processor has `config.return_attention_mask == False`, `attention_mask` should
+ **not** be passed to avoid degraded performance when doing batched inference. For such models
+ `input_values` should simply be padded with 0 and passed without `attention_mask`. Be aware that these
+ models also yield slightly different results depending on whether `input_values` is padded or not.
@@ -1268,7 +1269,7 @@ class WavLMModel(WavLMPreTrainedModel):
@add_start_docstrings(
- """WavLM Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC). """,
+ """WavLM Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC).""",
WAVLM_START_DOCSTRING,
)
# Copied from transformers.models.wav2vec2.modeling_wav2vec2.Wav2Vec2ForCTC with Wav2Vec2->WavLM, wav2vec2->wavlm, WAV_2_VEC_2->WAVLM
@@ -1317,7 +1318,9 @@ class WavLMForCTC(WavLMPreTrainedModel):
r"""
labels (`torch.LongTensor` of shape `(batch_size, target_length)`, *optional*):
Labels for connectionist temporal classification. Note that `target_length` has to be smaller or equal to
- the sequence length of the output logits. Indices are selected in `[-100, 0, ..., config.vocab_size - 1]`. All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size - 1]`.
+ the sequence length of the output logits. Indices are selected in `[-100, 0, ..., config.vocab_size - 1]`.
+ All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ...,
+ config.vocab_size - 1]`.
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1432,8 +1435,9 @@ class WavLMForSequenceClassification(WavLMPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1534,8 +1538,9 @@ class WavLMForAudioFrameClassification(WavLMPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1698,8 +1703,9 @@ class WavLMForXVector(WavLMPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
diff --git a/src/transformers/models/xlm/configuration_xlm.py b/src/transformers/models/xlm/configuration_xlm.py
index ba6dd8dfa0..a84a11002f 100644
--- a/src/transformers/models/xlm/configuration_xlm.py
+++ b/src/transformers/models/xlm/configuration_xlm.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" XLM configuration """
+""" XLM configuration"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging
@@ -36,13 +36,13 @@ XLM_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class XLMConfig(PretrainedConfig):
"""
- This is the configuration class to store the configuration of a [`XLMModel`] or a
- [`TFXLMModel`]. It is used to instantiate a XLM model according to the specified arguments,
- defining the model architecture. Instantiating a configuration with the defaults will yield a similar configuration
- to that of the [xlm-mlm-en-2048](https://huggingface.co/xlm-mlm-en-2048) architecture.
+ This is the configuration class to store the configuration of a [`XLMModel`] or a [`TFXLMModel`]. It is used to
+ instantiate a XLM model according to the specified arguments, defining the model architecture. Instantiating a
+ configuration with the defaults will yield a similar configuration to that of the
+ [xlm-mlm-en-2048](https://huggingface.co/xlm-mlm-en-2048) architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
vocab_size (`int`, *optional*, defaults to 30145):
@@ -72,8 +72,8 @@ class XLMConfig(PretrainedConfig):
The number of languages the model handles. Set to 1 for monolingual models.
use_lang_emb (`bool`, *optional*, defaults to `True`)
Whether to use language embeddings. Some models use additional language embeddings, see [the multilingual
- models page](http://huggingface.co/transformers/multilingual.html#xlm-language-embeddings) for
- information on how to use them.
+ models page](http://huggingface.co/transformers/multilingual.html#xlm-language-embeddings) for information
+ on how to use them.
max_position_embeddings (`int`, *optional*, defaults to 512):
The maximum sequence length that this model might ever be used with. Typically set this to something large
just in case (e.g., 512 or 1024 or 2048).
diff --git a/src/transformers/models/xlm/modeling_tf_xlm.py b/src/transformers/models/xlm/modeling_tf_xlm.py
index 4e097cf946..0f7cbed64c 100644
--- a/src/transformers/models/xlm/modeling_tf_xlm.py
+++ b/src/transformers/models/xlm/modeling_tf_xlm.py
@@ -563,12 +563,13 @@ class TFXLMWithLMHeadModelOutput(ModelOutput):
logits (`tf.Tensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -581,13 +582,13 @@ class TFXLMWithLMHeadModelOutput(ModelOutput):
XLM_START_DOCSTRING = r"""
- This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading or saving, resizing the input
- embeddings, pruning heads etc.)
+ This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use
- it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage
- and behavior.
+ This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it
+ as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and
+ behavior.
@@ -596,11 +597,11 @@ XLM_START_DOCSTRING = r"""
- having all inputs as keyword arguments (like PyTorch models), or
- having all inputs as a list, tuple or dict in the first positional arguments.
- This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all
- the tensors in the first argument of the model call function: `model(inputs)`.
+ This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all the
+ tensors in the first argument of the model call function: `model(inputs)`.
- If you choose this second option, there are three possibilities you can use to gather all the input Tensors in
- the first positional argument :
+ If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the
+ first positional argument :
- a single Tensor with `input_ids` only and nothing else: `model(inputs_ids)`
- a list of varying length with one or several input Tensors IN THE ORDER given in the docstring:
@@ -613,8 +614,7 @@ XLM_START_DOCSTRING = r"""
Parameters:
config ([`XLMConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
XLM_INPUTS_DOCSTRING = r"""
@@ -622,9 +622,8 @@ XLM_INPUTS_DOCSTRING = r"""
input_ids (`Numpy array` or `tf.Tensor` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`BertTokenizer`]. See
- [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for
- details.
+ Indices can be obtained using [`BertTokenizer`]. See [`PreTrainedTokenizer.__call__`] and
+ [`PreTrainedTokenizer.encode`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`Numpy array` or `tf.Tensor` of shape `({0})`, *optional*):
@@ -643,14 +642,16 @@ XLM_INPUTS_DOCSTRING = r"""
See usage examples detailed in the [multilingual documentation](../multilingual).
token_type_ids (`Numpy array` or `tf.Tensor` of shape `({0})`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`Numpy array` or `tf.Tensor` of shape `({0})`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
lengths (`tf.Tensor` or `Numpy array` of shape `(batch_size,)`, *optional*):
@@ -659,8 +660,8 @@ XLM_INPUTS_DOCSTRING = r"""
`[0, ..., input_ids.size(-1)]`.
cache (`Dict[str, tf.Tensor]`, *optional*):
Dictionary string to `torch.FloatTensor` that contains precomputed hidden states (key and values in the
- attention blocks) as computed by the model (see `cache` output below). Can be used to speed up
- sequential decoding.
+ attention blocks) as computed by the model (see `cache` output below). Can be used to speed up sequential
+ decoding.
The dictionary object will be modified in-place during the forward pass to add newly computed
hidden-states.
@@ -671,9 +672,9 @@ XLM_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
inputs_embeds (`tf.Tensor` of shape `({0}, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the
@@ -683,8 +684,8 @@ XLM_INPUTS_DOCSTRING = r"""
more detail. This argument can be used only in eager mode, in graph mode the value in the config will be
used instead.
return_dict (`bool`, *optional*):
- Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This
- argument can be used in eager mode, in graph mode the value will always be set to True.
+ Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple. This argument can be used
+ in eager mode, in graph mode the value will always be set to True.
training (`bool`, *optional*, defaults to `False`):
Whether or not to use the model in training mode (some modules like dropout modules have different
behaviors between training and evaluation).
@@ -970,8 +971,9 @@ class TFXLMForSequenceClassification(TFXLMPreTrainedModel, TFSequenceClassificat
):
r"""
labels (`tf.Tensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
inputs = input_processing(
func=self.call,
@@ -1351,12 +1353,12 @@ class TFXLMForQuestionAnsweringSimple(TFXLMPreTrainedModel, TFQuestionAnsweringL
r"""
start_positions (`tf.Tensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the start of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
end_positions (`tf.Tensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the end of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
"""
inputs = input_processing(
func=self.call,
diff --git a/src/transformers/models/xlm/modeling_xlm.py b/src/transformers/models/xlm/modeling_xlm.py
index 07ec3f36d1..77e34b7926 100755
--- a/src/transformers/models/xlm/modeling_xlm.py
+++ b/src/transformers/models/xlm/modeling_xlm.py
@@ -287,12 +287,13 @@ class XLMForQuestionAnsweringOutput(ModelOutput):
cls_logits (`torch.FloatTensor` of shape `(batch_size,)`, *optional*, returned if `start_positions` or `end_positions` is not provided):
Log probabilities for the `is_impossible` label of the answers.
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -310,19 +311,18 @@ class XLMForQuestionAnsweringOutput(ModelOutput):
XLM_START_DOCSTRING = r"""
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
Parameters:
config ([`XLMConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
XLM_INPUTS_DOCSTRING = r"""
@@ -330,9 +330,8 @@ XLM_INPUTS_DOCSTRING = r"""
input_ids (`torch.LongTensor` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`XLMTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`XLMTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
@@ -351,14 +350,16 @@ XLM_INPUTS_DOCSTRING = r"""
See usage examples detailed in the [multilingual documentation](../multilingual).
token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`.
+ Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+ config.max_position_embeddings - 1]`.
[What are position IDs?](../glossary#position-ids)
lengths (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
@@ -367,8 +368,8 @@ XLM_INPUTS_DOCSTRING = r"""
`[0, ..., input_ids.size(-1)]`.
cache (`Dict[str, torch.FloatTensor]`, *optional*):
Dictionary string to `torch.FloatTensor` that contains precomputed hidden states (key and values in the
- attention blocks) as computed by the model (see `cache` output below). Can be used to speed up
- sequential decoding.
+ attention blocks) as computed by the model (see `cache` output below). Can be used to speed up sequential
+ decoding.
The dictionary object will be modified in-place during the forward pass to add newly computed
hidden-states.
@@ -379,9 +380,9 @@ XLM_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -734,8 +735,8 @@ class XLMWithLMHeadModel(XLMPreTrainedModel):
r"""
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
- `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to
- `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
+ `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
+ are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -812,8 +813,9 @@ class XLMForSequenceClassification(XLMPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -914,12 +916,12 @@ class XLMForQuestionAnsweringSimple(XLMPreTrainedModel):
r"""
start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the start of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the end of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1017,12 +1019,12 @@ class XLMForQuestionAnswering(XLMPreTrainedModel):
r"""
start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the start of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the end of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
is_impossible (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels whether a question has an answer or no answer (SQuAD 2.0)
cls_index (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
@@ -1231,7 +1233,8 @@ class XLMForMultipleChoice(XLMPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
+ Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
+ num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
`input_ids` above)
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
diff --git a/src/transformers/models/xlm/tokenization_xlm.py b/src/transformers/models/xlm/tokenization_xlm.py
index 08e8604a7b..7519a514c9 100644
--- a/src/transformers/models/xlm/tokenization_xlm.py
+++ b/src/transformers/models/xlm/tokenization_xlm.py
@@ -534,14 +534,14 @@ class XLMTokenizer(PreTrainedTokenizer):
- Moses preprocessing and tokenization for most supported languages.
- Language specific tokenization for Chinese (Jieba), Japanese (KyTea) and Thai (PyThaiNLP).
- Optionally lowercases and normalizes all inputs text.
- - The arguments `special_tokens` and the function `set_special_tokens`, can be used to add additional symbols
- (like "__classify__") to a vocabulary.
- - The `lang2id` attribute maps the languages supported by the model with their IDs if provided (automatically
- set for pretrained vocabularies).
+ - The arguments `special_tokens` and the function `set_special_tokens`, can be used to add additional symbols (like
+ "__classify__") to a vocabulary.
+ - The `lang2id` attribute maps the languages supported by the model with their IDs if provided (automatically set
+ for pretrained vocabularies).
- The `id2lang` attributes does reverse mapping if provided (automatically set for pretrained vocabularies).
- This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods.
- Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
+ this superclass for more information regarding those methods.
Args:
vocab_file (`str`):
@@ -767,11 +767,8 @@ class XLMTokenizer(PreTrainedTokenizer):
::
- git clone git@github.com:neubig/kytea.git && cd kytea
- autoreconf -i
- ./configure --prefix=$HOME/local
- make && make install
- pip install kytea
+ git clone git@github.com:neubig/kytea.git && cd kytea autoreconf -i ./configure --prefix=$HOME/local
+ make && make install pip install kytea
- [jieba](https://github.com/fxsjy/jieba): Chinese tokenizer (*)
- Install with `pip install jieba`
@@ -938,8 +935,7 @@ class XLMTokenizer(PreTrainedTokenizer):
Optional second list of IDs for sequence pairs.
Returns:
- `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given
- sequence(s).
+ `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
"""
sep = [self.sep_token_id]
cls = [self.cls_token_id]
diff --git a/src/transformers/models/xlm_prophetnet/configuration_xlm_prophetnet.py b/src/transformers/models/xlm_prophetnet/configuration_xlm_prophetnet.py
index 27d6117708..ccf1130d7a 100644
--- a/src/transformers/models/xlm_prophetnet/configuration_xlm_prophetnet.py
+++ b/src/transformers/models/xlm_prophetnet/configuration_xlm_prophetnet.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" XLM-ProphetNet model configuration """
+""" XLM-ProphetNet model configuration"""
from ...utils import logging
@@ -28,8 +28,8 @@ XLM_PROPHETNET_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class XLMProphetNetConfig(ProphetNetConfig):
"""
- This class overrides [`ProphetNetConfig`]. Please check the superclass for the appropriate
- documentation alongside usage examples.
+ This class overrides [`ProphetNetConfig`]. Please check the superclass for the appropriate documentation alongside
+ usage examples.
"""
model_type = "xlm-prophetnet"
diff --git a/src/transformers/models/xlm_prophetnet/modeling_xlm_prophetnet.py b/src/transformers/models/xlm_prophetnet/modeling_xlm_prophetnet.py
index 99d3156210..c272de5d64 100644
--- a/src/transformers/models/xlm_prophetnet/modeling_xlm_prophetnet.py
+++ b/src/transformers/models/xlm_prophetnet/modeling_xlm_prophetnet.py
@@ -37,8 +37,8 @@ XLM_PROPHETNET_PRETRAINED_MODEL_ARCHIVE_LIST = [
class XLMProphetNetEncoder(ProphetNetEncoder):
r"""
- This class overrides [`ProphetNetEncoder`]. Please check the superclass for the appropriate
- documentation alongside usage examples.
+ This class overrides [`ProphetNetEncoder`]. Please check the superclass for the appropriate documentation alongside
+ usage examples.
Example:
@@ -60,8 +60,8 @@ class XLMProphetNetEncoder(ProphetNetEncoder):
class XLMProphetNetDecoder(ProphetNetDecoder):
r"""
- This class overrides [`ProphetNetDecoder`]. Please check the superclass for the appropriate
- documentation alongside usage examples.
+ This class overrides [`ProphetNetDecoder`]. Please check the superclass for the appropriate documentation alongside
+ usage examples.
Example:
@@ -83,8 +83,8 @@ class XLMProphetNetDecoder(ProphetNetDecoder):
class XLMProphetNetModel(ProphetNetModel):
r"""
- This class overrides [`ProphetNetModel`]. Please check the superclass for the appropriate
- documentation alongside usage examples.
+ This class overrides [`ProphetNetModel`]. Please check the superclass for the appropriate documentation alongside
+ usage examples.
Example:
@@ -107,8 +107,8 @@ class XLMProphetNetModel(ProphetNetModel):
class XLMProphetNetForConditionalGeneration(ProphetNetForConditionalGeneration):
r"""
- This class overrides [`ProphetNetForConditionalGeneration`]. Please check the superclass for the
- appropriate documentation alongside usage examples.
+ This class overrides [`ProphetNetForConditionalGeneration`]. Please check the superclass for the appropriate
+ documentation alongside usage examples.
Example:
@@ -131,8 +131,8 @@ class XLMProphetNetForConditionalGeneration(ProphetNetForConditionalGeneration):
class XLMProphetNetForCausalLM(ProphetNetForCausalLM):
r"""
- This class overrides [`ProphetNetForCausalLM`]. Please check the superclass for the appropriate
- documentation alongside usage examples.
+ This class overrides [`ProphetNetForCausalLM`]. Please check the superclass for the appropriate documentation
+ alongside usage examples.
Example:
diff --git a/src/transformers/models/xlm_prophetnet/tokenization_xlm_prophetnet.py b/src/transformers/models/xlm_prophetnet/tokenization_xlm_prophetnet.py
index dda9f80560..9004433acf 100644
--- a/src/transformers/models/xlm_prophetnet/tokenization_xlm_prophetnet.py
+++ b/src/transformers/models/xlm_prophetnet/tokenization_xlm_prophetnet.py
@@ -59,8 +59,8 @@ class XLMProphetNetTokenizer(PreTrainedTokenizer):
Adapted from [`RobertaTokenizer`] and [`XLNetTokenizer`]. Based on
[SentencePiece](https://github.com/google/sentencepiece).
- This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods.
- Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
+ this superclass for more information regarding those methods.
Args:
vocab_file (`str`):
@@ -80,8 +80,8 @@ class XLMProphetNetTokenizer(PreTrainedTokenizer):
- When building a sequence using special tokens, this is not the token that is used for the end of
- sequence. The token used is the `sep_token`.
+ When building a sequence using special tokens, this is not the token that is used for the end of sequence.
+ The token used is the `sep_token`.
@@ -103,7 +103,9 @@ class XLMProphetNetTokenizer(PreTrainedTokenizer):
additional_special_tokens (`List[str]`, *optional*, defaults to `["NOTUSED", "NOTUSED"]`):
Additional special tokens used by the tokenizer.
sp_model_kwargs (`dict`, *optional*):
- Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things, to set:
+ Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for
+ SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things,
+ to set:
- `enable_sampling`: Enable subword regularization.
- `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout.
diff --git a/src/transformers/models/xlm_roberta/configuration_xlm_roberta.py b/src/transformers/models/xlm_roberta/configuration_xlm_roberta.py
index 6ee2d52e86..568aedcfca 100644
--- a/src/transformers/models/xlm_roberta/configuration_xlm_roberta.py
+++ b/src/transformers/models/xlm_roberta/configuration_xlm_roberta.py
@@ -13,7 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" XLM-RoBERTa configuration """
+""" XLM-RoBERTa configuration"""
from collections import OrderedDict
from typing import Mapping
@@ -36,8 +36,8 @@ XLM_ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class XLMRobertaConfig(RobertaConfig):
"""
- This class overrides [`RobertaConfig`]. Please check the superclass for the appropriate
- documentation alongside usage examples.
+ This class overrides [`RobertaConfig`]. Please check the superclass for the appropriate documentation alongside
+ usage examples.
"""
model_type = "xlm-roberta"
diff --git a/src/transformers/models/xlm_roberta/modeling_tf_xlm_roberta.py b/src/transformers/models/xlm_roberta/modeling_tf_xlm_roberta.py
index 1e4cdff3fa..c699676672 100644
--- a/src/transformers/models/xlm_roberta/modeling_tf_xlm_roberta.py
+++ b/src/transformers/models/xlm_roberta/modeling_tf_xlm_roberta.py
@@ -13,7 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" TF 2.0 XLM-RoBERTa model. """
+""" TF 2.0 XLM-RoBERTa model."""
from ...file_utils import add_start_docstrings
from ...utils import logging
@@ -38,13 +38,13 @@ TF_XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST = [
XLM_ROBERTA_START_DOCSTRING = r"""
- This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading or saving, resizing the input
- embeddings, pruning heads etc.)
+ This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use
- it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage
- and behavior.
+ This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it
+ as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and
+ behavior.
@@ -53,11 +53,11 @@ XLM_ROBERTA_START_DOCSTRING = r"""
- having all inputs as keyword arguments (like PyTorch models), or
- having all inputs as a list, tuple or dict in the first positional arguments.
- This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all
- the tensors in the first argument of the model call function: `model(inputs)`.
+ This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all the
+ tensors in the first argument of the model call function: `model(inputs)`.
- If you choose this second option, there are three possibilities you can use to gather all the input Tensors in
- the first positional argument :
+ If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the
+ first positional argument :
- a single Tensor with `input_ids` only and nothing else: `model(inputs_ids)`
- a list of varying length with one or several input Tensors IN THE ORDER given in the docstring:
@@ -70,8 +70,7 @@ XLM_ROBERTA_START_DOCSTRING = r"""
Parameters:
config ([`XLMRobertaConfig`]): Model configuration class with all the parameters of the
model. Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
@@ -81,8 +80,8 @@ XLM_ROBERTA_START_DOCSTRING = r"""
)
class TFXLMRobertaModel(TFRobertaModel):
"""
- This class overrides [`TFRobertaModel`]. Please check the superclass for the appropriate
- documentation alongside usage examples.
+ This class overrides [`TFRobertaModel`]. Please check the superclass for the appropriate documentation alongside
+ usage examples.
"""
config_class = XLMRobertaConfig
@@ -94,21 +93,21 @@ class TFXLMRobertaModel(TFRobertaModel):
)
class XLMRobertaForCausalLM(TFRobertaForCausalLM):
"""
- This class overrides [`TFRobertaForCausalLM`]. Please check the superclass for the appropriate
- documentation alongside usage examples.
+ This class overrides [`TFRobertaForCausalLM`]. Please check the superclass for the appropriate documentation
+ alongside usage examples.
"""
config_class = XLMRobertaConfig
@add_start_docstrings(
- """XLM-RoBERTa Model with a `language modeling` head on top. """,
+ """XLM-RoBERTa Model with a `language modeling` head on top.""",
XLM_ROBERTA_START_DOCSTRING,
)
class TFXLMRobertaForMaskedLM(TFRobertaForMaskedLM):
"""
- This class overrides [`TFRobertaForMaskedLM`]. Please check the superclass for the appropriate
- documentation alongside usage examples.
+ This class overrides [`TFRobertaForMaskedLM`]. Please check the superclass for the appropriate documentation
+ alongside usage examples.
"""
config_class = XLMRobertaConfig
@@ -123,8 +122,8 @@ class TFXLMRobertaForMaskedLM(TFRobertaForMaskedLM):
)
class TFXLMRobertaForSequenceClassification(TFRobertaForSequenceClassification):
"""
- This class overrides [`TFRobertaForSequenceClassification`]. Please check the superclass for the
- appropriate documentation alongside usage examples.
+ This class overrides [`TFRobertaForSequenceClassification`]. Please check the superclass for the appropriate
+ documentation alongside usage examples.
"""
config_class = XLMRobertaConfig
@@ -139,8 +138,8 @@ class TFXLMRobertaForSequenceClassification(TFRobertaForSequenceClassification):
)
class TFXLMRobertaForTokenClassification(TFRobertaForTokenClassification):
"""
- This class overrides [`TFRobertaForTokenClassification`]. Please check the superclass for the
- appropriate documentation alongside usage examples.
+ This class overrides [`TFRobertaForTokenClassification`]. Please check the superclass for the appropriate
+ documentation alongside usage examples.
"""
config_class = XLMRobertaConfig
@@ -155,8 +154,8 @@ layers on top of the hidden-states output to compute `span start logits` and `sp
)
class TFXLMRobertaForQuestionAnswering(TFRobertaForQuestionAnswering):
"""
- This class overrides [`TFRobertaForQuestionAnsweringSimple`]. Please check the superclass for
- the appropriate documentation alongside usage examples.
+ This class overrides [`TFRobertaForQuestionAnsweringSimple`]. Please check the superclass for the appropriate
+ documentation alongside usage examples.
"""
config_class = XLMRobertaConfig
@@ -171,8 +170,8 @@ class TFXLMRobertaForQuestionAnswering(TFRobertaForQuestionAnswering):
)
class TFXLMRobertaForMultipleChoice(TFRobertaForMultipleChoice):
"""
- This class overrides [`TFRobertaForMultipleChoice`]. Please check the superclass for the
- appropriate documentation alongside usage examples.
+ This class overrides [`TFRobertaForMultipleChoice`]. Please check the superclass for the appropriate documentation
+ alongside usage examples.
"""
config_class = XLMRobertaConfig
diff --git a/src/transformers/models/xlm_roberta/modeling_xlm_roberta.py b/src/transformers/models/xlm_roberta/modeling_xlm_roberta.py
index 6936503c54..2221502ff5 100644
--- a/src/transformers/models/xlm_roberta/modeling_xlm_roberta.py
+++ b/src/transformers/models/xlm_roberta/modeling_xlm_roberta.py
@@ -13,7 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-"""PyTorch XLM-RoBERTa model. """
+"""PyTorch XLM-RoBERTa model."""
from ...file_utils import add_start_docstrings
from ...utils import logging
@@ -44,19 +44,18 @@ XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST = [
XLM_ROBERTA_START_DOCSTRING = r"""
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
Parameters:
config ([`XLMRobertaConfig`]): Model configuration class with all the parameters of the
model. Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
@@ -66,8 +65,8 @@ XLM_ROBERTA_START_DOCSTRING = r"""
)
class XLMRobertaModel(RobertaModel):
"""
- This class overrides [`RobertaModel`]. Please check the superclass for the appropriate
- documentation alongside usage examples.
+ This class overrides [`RobertaModel`]. Please check the superclass for the appropriate documentation alongside
+ usage examples.
"""
config_class = XLMRobertaConfig
@@ -79,21 +78,21 @@ class XLMRobertaModel(RobertaModel):
)
class XLMRobertaForCausalLM(RobertaForCausalLM):
"""
- This class overrides [`RobertaForCausalLM`]. Please check the superclass for the appropriate
- documentation alongside usage examples.
+ This class overrides [`RobertaForCausalLM`]. Please check the superclass for the appropriate documentation
+ alongside usage examples.
"""
config_class = XLMRobertaConfig
@add_start_docstrings(
- """XLM-RoBERTa Model with a `language modeling` head on top. """,
+ """XLM-RoBERTa Model with a `language modeling` head on top.""",
XLM_ROBERTA_START_DOCSTRING,
)
class XLMRobertaForMaskedLM(RobertaForMaskedLM):
"""
- This class overrides [`RobertaForMaskedLM`]. Please check the superclass for the appropriate
- documentation alongside usage examples.
+ This class overrides [`RobertaForMaskedLM`]. Please check the superclass for the appropriate documentation
+ alongside usage examples.
"""
config_class = XLMRobertaConfig
@@ -108,8 +107,8 @@ class XLMRobertaForMaskedLM(RobertaForMaskedLM):
)
class XLMRobertaForSequenceClassification(RobertaForSequenceClassification):
"""
- This class overrides [`RobertaForSequenceClassification`]. Please check the superclass for the
- appropriate documentation alongside usage examples.
+ This class overrides [`RobertaForSequenceClassification`]. Please check the superclass for the appropriate
+ documentation alongside usage examples.
"""
config_class = XLMRobertaConfig
@@ -124,8 +123,8 @@ class XLMRobertaForSequenceClassification(RobertaForSequenceClassification):
)
class XLMRobertaForMultipleChoice(RobertaForMultipleChoice):
"""
- This class overrides [`RobertaForMultipleChoice`]. Please check the superclass for the
- appropriate documentation alongside usage examples.
+ This class overrides [`RobertaForMultipleChoice`]. Please check the superclass for the appropriate documentation
+ alongside usage examples.
"""
config_class = XLMRobertaConfig
@@ -140,8 +139,8 @@ class XLMRobertaForMultipleChoice(RobertaForMultipleChoice):
)
class XLMRobertaForTokenClassification(RobertaForTokenClassification):
"""
- This class overrides [`RobertaForTokenClassification`]. Please check the superclass for the
- appropriate documentation alongside usage examples.
+ This class overrides [`RobertaForTokenClassification`]. Please check the superclass for the appropriate
+ documentation alongside usage examples.
"""
config_class = XLMRobertaConfig
@@ -156,8 +155,8 @@ class XLMRobertaForTokenClassification(RobertaForTokenClassification):
)
class XLMRobertaForQuestionAnswering(RobertaForQuestionAnswering):
"""
- This class overrides [`RobertaForQuestionAnswering`]. Please check the superclass for the
- appropriate documentation alongside usage examples.
+ This class overrides [`RobertaForQuestionAnswering`]. Please check the superclass for the appropriate documentation
+ alongside usage examples.
"""
config_class = XLMRobertaConfig
diff --git a/src/transformers/models/xlm_roberta/tokenization_xlm_roberta.py b/src/transformers/models/xlm_roberta/tokenization_xlm_roberta.py
index 80bd7b419f..824b8279e9 100644
--- a/src/transformers/models/xlm_roberta/tokenization_xlm_roberta.py
+++ b/src/transformers/models/xlm_roberta/tokenization_xlm_roberta.py
@@ -57,8 +57,8 @@ class XLMRobertaTokenizer(PreTrainedTokenizer):
Adapted from [`RobertaTokenizer`] and [`XLNetTokenizer`]. Based on
[SentencePiece](https://github.com/google/sentencepiece).
- This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods.
- Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
+ this superclass for more information regarding those methods.
Args:
vocab_file (`str`):
@@ -78,8 +78,8 @@ class XLMRobertaTokenizer(PreTrainedTokenizer):
- When building a sequence using special tokens, this is not the token that is used for the end of
- sequence. The token used is the `sep_token`.
+ When building a sequence using special tokens, this is not the token that is used for the end of sequence.
+ The token used is the `sep_token`.
@@ -101,7 +101,9 @@ class XLMRobertaTokenizer(PreTrainedTokenizer):
additional_special_tokens (`List[str]`, *optional*, defaults to `["NOTUSED", "NOTUSED"]`):
Additional special tokens used by the tokenizer.
sp_model_kwargs (`dict`, *optional*):
- Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things, to set:
+ Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for
+ SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things,
+ to set:
- `enable_sampling`: Enable subword regularization.
- `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout.
diff --git a/src/transformers/models/xlm_roberta/tokenization_xlm_roberta_fast.py b/src/transformers/models/xlm_roberta/tokenization_xlm_roberta_fast.py
index 4c30c9cbb6..62663347f8 100644
--- a/src/transformers/models/xlm_roberta/tokenization_xlm_roberta_fast.py
+++ b/src/transformers/models/xlm_roberta/tokenization_xlm_roberta_fast.py
@@ -67,10 +67,11 @@ PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
class XLMRobertaTokenizerFast(PreTrainedTokenizerFast):
"""
Construct a "fast" XLM-RoBERTa tokenizer (backed by HuggingFace's *tokenizers* library). Adapted from
- [`RobertaTokenizer`] and [`XLNetTokenizer`]. Based on [BPE](https://huggingface.co/docs/tokenizers/python/latest/components.html?highlight=BPE#models).
+ [`RobertaTokenizer`] and [`XLNetTokenizer`]. Based on
+ [BPE](https://huggingface.co/docs/tokenizers/python/latest/components.html?highlight=BPE#models).
- This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main
- methods. Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should
+ refer to this superclass for more information regarding those methods.
Args:
vocab_file (`str`):
@@ -90,8 +91,8 @@ class XLMRobertaTokenizerFast(PreTrainedTokenizerFast):
- When building a sequence using special tokens, this is not the token that is used for the end of
- sequence. The token used is the `sep_token`.
+ When building a sequence using special tokens, this is not the token that is used for the end of sequence.
+ The token used is the `sep_token`.
diff --git a/src/transformers/models/xlnet/configuration_xlnet.py b/src/transformers/models/xlnet/configuration_xlnet.py
index 1029428669..bc6f0f6835 100644
--- a/src/transformers/models/xlnet/configuration_xlnet.py
+++ b/src/transformers/models/xlnet/configuration_xlnet.py
@@ -13,7 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" XLNet configuration """
+""" XLNet configuration"""
import warnings
@@ -31,19 +31,18 @@ XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP = {
class XLNetConfig(PretrainedConfig):
"""
- This is the configuration class to store the configuration of a [`XLNetModel`] or a
- [`TFXLNetModel`]. It is used to instantiate a XLNet model according to the specified arguments,
- defining the model architecture. Instantiating a configuration with the defaults will yield a similar configuration
- to that of the [xlnet-large-cased](https://huggingface.co/xlnet-large-cased) architecture.
+ This is the configuration class to store the configuration of a [`XLNetModel`] or a [`TFXLNetModel`]. It is used to
+ instantiate a XLNet model according to the specified arguments, defining the model architecture. Instantiating a
+ configuration with the defaults will yield a similar configuration to that of the
+ [xlnet-large-cased](https://huggingface.co/xlnet-large-cased) architecture.
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model
- outputs. Read the documentation from [`PretrainedConfig`] for more information.
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
Args:
vocab_size (`int`, *optional*, defaults to 32000):
Vocabulary size of the XLNet model. Defines the number of different tokens that can be represented by the
- `inputs_ids` passed when calling [`XLNetModel`] or
- [`TFXLNetModel`].
+ `inputs_ids` passed when calling [`XLNetModel`] or [`TFXLNetModel`].
d_model (`int`, *optional*, defaults to 1024):
Dimensionality of the encoder layers and the pooler layer.
n_layer (`int`, *optional*, defaults to 24):
@@ -53,8 +52,8 @@ class XLNetConfig(PretrainedConfig):
d_inner (`int`, *optional*, defaults to 4096):
Dimensionality of the "intermediate" (often named feed-forward) layer in the Transformer encoder.
ff_activation (`str` or `Callable`, *optional*, defaults to `"gelu"`):
- The non-linear activation function (function or string) in the If string, `"gelu"`, `"relu"`,
- `"silu"` and `"gelu_new"` are supported.
+ The non-linear activation function (function or string) in the If string, `"gelu"`, `"relu"`, `"silu"` and
+ `"gelu_new"` are supported.
untie_r (`bool`, *optional*, defaults to `True`):
Whether or not to untie relative position biases
attn_type (`str`, *optional*, defaults to `"bi"`):
@@ -67,12 +66,13 @@ class XLNetConfig(PretrainedConfig):
The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
mem_len (`int` or `None`, *optional*):
The number of tokens to cache. The key/value pairs that have already been pre-computed in a previous
- forward pass won't be re-computed. See the [quickstart](https://huggingface.co/transformers/quickstart.html#using-the-past) for more information.
+ forward pass won't be re-computed. See the
+ [quickstart](https://huggingface.co/transformers/quickstart.html#using-the-past) for more information.
reuse_len (`int`, *optional*):
The number of tokens in the current batch to be cached and reused in the future.
bi_data (`bool`, *optional*, defaults to `False`):
- Whether or not to use bidirectional input pipeline. Usually set to `True` during pretraining and
- `False` during finetuning.
+ Whether or not to use bidirectional input pipeline. Usually set to `True` during pretraining and `False`
+ during finetuning.
clamp_len (`int`, *optional*, defaults to -1):
Clamp all relative distances larger than clamp_len. Setting this attribute to -1 means no clamping.
same_length (`bool`, *optional*, defaults to `False`):
@@ -114,10 +114,12 @@ class XLNetConfig(PretrainedConfig):
- For pretraining, it is recommended to set `use_mems_train` to `True`. For fine-tuning, it is
- recommended to set `use_mems_train` to `False` as discussed [here](https://github.com/zihangdai/xlnet/issues/41#issuecomment-505102587). If `use_mems_train` is set
- to `True`, one has to make sure that the train batches are correctly pre-processed, *e.g.*
- `batch_1 = [[This line is], [This is the]]` and `batch_2 = [[ the first line], [ second line]]` and that all batches are of equal size.
+ For pretraining, it is recommended to set `use_mems_train` to `True`. For fine-tuning, it is recommended to
+ set `use_mems_train` to `False` as discussed
+ [here](https://github.com/zihangdai/xlnet/issues/41#issuecomment-505102587). If `use_mems_train` is set to
+ `True`, one has to make sure that the train batches are correctly pre-processed, *e.g.* `batch_1 = [[This
+ line is], [This is the]]` and `batch_2 = [[ the first line], [ second line]]` and that all batches are of
+ equal size.
diff --git a/src/transformers/models/xlnet/modeling_tf_xlnet.py b/src/transformers/models/xlnet/modeling_tf_xlnet.py
index 02a7629eba..1ea87aa3a3 100644
--- a/src/transformers/models/xlnet/modeling_tf_xlnet.py
+++ b/src/transformers/models/xlnet/modeling_tf_xlnet.py
@@ -832,19 +832,20 @@ class TFXLNetModelOutput(ModelOutput):
last_hidden_state (`tf.Tensor` of shape `(batch_size, num_predict, hidden_size)`):
Sequence of hidden-states at the last layer of the model.
- `num_predict` corresponds to `target_mapping.shape[1]`. If `target_mapping` is `None`, then
- `num_predict` corresponds to `sequence_length`.
+ `num_predict` corresponds to `target_mapping.shape[1]`. If `target_mapping` is `None`, then `num_predict`
+ corresponds to `sequence_length`.
mems (`List[tf.Tensor]` of length `config.n_layers`):
- Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding.
- The token ids which have their past given to this model should not be passed as `input_ids` as they
- have already been computed.
+ Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The
+ token ids which have their past given to this model should not be passed as `input_ids` as they have
+ already been computed.
hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -867,19 +868,20 @@ class TFXLNetLMHeadModelOutput(ModelOutput):
logits (`tf.Tensor` of shape `(batch_size, num_predict, config.vocab_size)`):
Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
- `num_predict` corresponds to `target_mapping.shape[1]`. If `target_mapping` is `None`, then
- `num_predict` corresponds to `sequence_length`.
+ `num_predict` corresponds to `target_mapping.shape[1]`. If `target_mapping` is `None`, then `num_predict`
+ corresponds to `sequence_length`.
mems (`List[tf.Tensor]` of length `config.n_layers`):
- Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding.
- The token ids which have their past given to this model should not be passed as `input_ids` as they
- have already been computed.
+ Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The
+ token ids which have their past given to this model should not be passed as `input_ids` as they have
+ already been computed.
hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -903,16 +905,17 @@ class TFXLNetForSequenceClassificationOutput(ModelOutput):
logits (`tf.Tensor` of shape `(batch_size, config.num_labels)`):
Classification (or regression if config.num_labels==1) scores (before SoftMax).
mems (`List[tf.Tensor]` of length `config.n_layers`):
- Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding.
- The token ids which have their past given to this model should not be passed as `input_ids` as they
- have already been computed.
+ Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The
+ token ids which have their past given to this model should not be passed as `input_ids` as they have
+ already been computed.
hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -936,16 +939,17 @@ class TFXLNetForTokenClassificationOutput(ModelOutput):
logits (`tf.Tensor` of shape `(batch_size, sequence_length, config.num_labels)`):
Classification scores (before SoftMax).
mems (`List[tf.Tensor]` of length `config.n_layers`):
- Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding.
- The token ids which have their past given to this model should not be passed as `input_ids` as they
- have already been computed.
+ Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The
+ token ids which have their past given to this model should not be passed as `input_ids` as they have
+ already been computed.
hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -971,16 +975,17 @@ class TFXLNetForMultipleChoiceOutput(ModelOutput):
Classification scores (before SoftMax).
mems (`List[tf.Tensor]` of length `config.n_layers`):
- Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding.
- The token ids which have their past given to this model should not be passed as `input_ids` as they
- have already been computed.
+ Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The
+ token ids which have their past given to this model should not be passed as `input_ids` as they have
+ already been computed.
hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -1006,16 +1011,17 @@ class TFXLNetForQuestionAnsweringSimpleOutput(ModelOutput):
end_logits (`tf.Tensor` of shape `(batch_size, sequence_length,)`):
Span-end scores (before SoftMax).
mems (`List[tf.Tensor]` of length `config.n_layers`):
- Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding.
- The token ids which have their past given to this model should not be passed as `input_ids` as they
- have already been computed.
+ Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The
+ token ids which have their past given to this model should not be passed as `input_ids` as they have
+ already been computed.
hidden_states (`tuple(tf.Tensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of
- shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `tf.Tensor` (one for the output of the embeddings + one for the output of each layer) of shape
+ `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(tf.Tensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `tf.Tensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -1031,13 +1037,13 @@ class TFXLNetForQuestionAnsweringSimpleOutput(ModelOutput):
XLNET_START_DOCSTRING = r"""
- This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the
- generic methods the library implements for all its model (such as downloading or saving, resizing the input
- embeddings, pruning heads etc.)
+ This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use
- it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage
- and behavior.
+ This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it
+ as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and
+ behavior.
@@ -1046,11 +1052,11 @@ XLNET_START_DOCSTRING = r"""
- having all inputs as keyword arguments (like PyTorch models), or
- having all inputs as a list, tuple or dict in the first positional arguments.
- This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all
- the tensors in the first argument of the model call function: `model(inputs)`.
+ This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all the
+ tensors in the first argument of the model call function: `model(inputs)`.
- If you choose this second option, there are three possibilities you can use to gather all the input Tensors in
- the first positional argument :
+ If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the
+ first positional argument :
- a single Tensor with `input_ids` only and nothing else: `model(inputs_ids)`
- a list of varying length with one or several input Tensors IN THE ORDER given in the docstring:
@@ -1063,8 +1069,7 @@ XLNET_START_DOCSTRING = r"""
Parameters:
config ([`XLNetConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
XLNET_INPUTS_DOCSTRING = r"""
@@ -1072,9 +1077,8 @@ XLNET_INPUTS_DOCSTRING = r"""
input_ids (`torch.LongTensor` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`XLNetTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`XLNetTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
@@ -1086,8 +1090,8 @@ XLNET_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
mems (`List[torch.FloatTensor]` of length `config.n_layers`):
Contains pre-computed hidden-states (see `mems` output below) . Can be used to speed up sequential
- decoding. The token ids which have their past given to this model should not be passed as `input_ids`
- as they have already been computed.
+ decoding. The token ids which have their past given to this model should not be passed as `input_ids` as
+ they have already been computed.
`use_mems` has to be set to `True` to make use of `mems`.
perm_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length, sequence_length)`, *optional*):
@@ -1099,19 +1103,20 @@ XLNET_INPUTS_DOCSTRING = r"""
If not set, each token attends to all the others (full bidirectional attention). Only used during
pretraining (to define factorization order) or for sequential decoding (generation).
target_mapping (`torch.FloatTensor` of shape `(batch_size, num_predict, sequence_length)`, *optional*):
- Mask to indicate the output tokens to use. If `target_mapping[k, i, j] = 1`, the i-th predict in batch k
- is on the j-th token. Only used during pretraining for partial prediction or for sequential decoding
+ Mask to indicate the output tokens to use. If `target_mapping[k, i, j] = 1`, the i-th predict in batch k is
+ on the j-th token. Only used during pretraining for partial prediction or for sequential decoding
(generation).
token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
input_mask (`torch.FloatTensor` of shape `{0}`, *optional*):
- Mask to avoid performing attention on padding token indices. Negative of `attention_mask`, i.e. with 0
- for real tokens and 1 for padding which is kept for compatibility with the original code base.
+ Mask to avoid performing attention on padding token indices. Negative of `attention_mask`, i.e. with 0 for
+ real tokens and 1 for padding which is kept for compatibility with the original code base.
Mask values selected in `[0, 1]`:
@@ -1126,9 +1131,9 @@ XLNET_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -1304,7 +1309,8 @@ class TFXLNetLMHeadModel(TFXLNetPreTrainedModel, TFCausalLanguageModelingLoss):
):
r"""
labels (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
- Labels for computing the cross entropy classification loss. Indices should be in `[0, ..., config.vocab_size - 1]`.
+ Labels for computing the cross entropy classification loss. Indices should be in `[0, ...,
+ config.vocab_size - 1]`.
Return:
@@ -1445,8 +1451,9 @@ class TFXLNetForSequenceClassification(TFXLNetPreTrainedModel, TFSequenceClassif
):
r"""
labels (`tf.Tensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
inputs = input_processing(
func=self.call,
@@ -1570,8 +1577,8 @@ class TFXLNetForMultipleChoice(TFXLNetPreTrainedModel, TFMultipleChoiceLoss):
):
r"""
labels (`tf.Tensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices]` where `num_choices` is the size of the second dimension of the input tensors. (See
- `input_ids` above)
+ Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices]`
+ where `num_choices` is the size of the second dimension of the input tensors. (See `input_ids` above)
"""
inputs = input_processing(
@@ -1826,12 +1833,12 @@ class TFXLNetForQuestionAnsweringSimple(TFXLNetPreTrainedModel, TFQuestionAnswer
r"""
start_positions (`tf.Tensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the start of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
end_positions (`tf.Tensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the end of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
"""
inputs = input_processing(
func=self.call,
diff --git a/src/transformers/models/xlnet/modeling_xlnet.py b/src/transformers/models/xlnet/modeling_xlnet.py
index 4556aef83f..828af81cd8 100755
--- a/src/transformers/models/xlnet/modeling_xlnet.py
+++ b/src/transformers/models/xlnet/modeling_xlnet.py
@@ -591,19 +591,20 @@ class XLNetModelOutput(ModelOutput):
last_hidden_state (`torch.FloatTensor` of shape `(batch_size, num_predict, hidden_size)`):
Sequence of hidden-states at the last layer of the model.
- `num_predict` corresponds to `target_mapping.shape[1]`. If `target_mapping` is `None`, then
- `num_predict` corresponds to `sequence_length`.
+ `num_predict` corresponds to `target_mapping.shape[1]`. If `target_mapping` is `None`, then `num_predict`
+ corresponds to `sequence_length`.
mems (`List[torch.FloatTensor]` of length `config.n_layers`):
- Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding.
- The token ids which have their past given to this model should not be passed as `input_ids` as they
- have already been computed.
+ Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The
+ token ids which have their past given to this model should not be passed as `input_ids` as they have
+ already been computed.
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -626,19 +627,20 @@ class XLNetLMHeadModelOutput(ModelOutput):
logits (`torch.FloatTensor` of shape `(batch_size, num_predict, config.vocab_size)`):
Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
- `num_predict` corresponds to `target_mapping.shape[1]`. If `target_mapping` is `None`, then
- `num_predict` corresponds to `sequence_length`.
+ `num_predict` corresponds to `target_mapping.shape[1]`. If `target_mapping` is `None`, then `num_predict`
+ corresponds to `sequence_length`.
mems (`List[torch.FloatTensor]` of length `config.n_layers`):
- Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding.
- The token ids which have their past given to this model should not be passed as `input_ids` as they
- have already been computed.
+ Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The
+ token ids which have their past given to this model should not be passed as `input_ids` as they have
+ already been computed.
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -662,16 +664,17 @@ class XLNetForSequenceClassificationOutput(ModelOutput):
logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
Classification (or regression if config.num_labels==1) scores (before SoftMax).
mems (`List[torch.FloatTensor]` of length `config.n_layers`):
- Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding.
- The token ids which have their past given to this model should not be passed as `input_ids` as they
- have already been computed.
+ Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The
+ token ids which have their past given to this model should not be passed as `input_ids` as they have
+ already been computed.
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -695,16 +698,17 @@ class XLNetForTokenClassificationOutput(ModelOutput):
logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.num_labels)`):
Classification scores (before SoftMax).
mems (`List[torch.FloatTensor]` of length `config.n_layers`):
- Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding.
- The token ids which have their past given to this model should not be passed as `input_ids` as they
- have already been computed.
+ Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The
+ token ids which have their past given to this model should not be passed as `input_ids` as they have
+ already been computed.
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -730,16 +734,17 @@ class XLNetForMultipleChoiceOutput(ModelOutput):
Classification scores (before SoftMax).
mems (`List[torch.FloatTensor]` of length `config.n_layers`):
- Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding.
- The token ids which have their past given to this model should not be passed as `input_ids` as they
- have already been computed.
+ Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The
+ token ids which have their past given to this model should not be passed as `input_ids` as they have
+ already been computed.
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -765,16 +770,17 @@ class XLNetForQuestionAnsweringSimpleOutput(ModelOutput):
end_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length,)`):
Span-end scores (before SoftMax).
mems (`List[torch.FloatTensor]` of length `config.n_layers`):
- Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding.
- The token ids which have their past given to this model should not be passed as `input_ids` as they
- have already been computed.
+ Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The
+ token ids which have their past given to this model should not be passed as `input_ids` as they have
+ already been computed.
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -809,16 +815,17 @@ class XLNetForQuestionAnsweringOutput(ModelOutput):
cls_logits (`torch.FloatTensor` of shape `(batch_size,)`, *optional*, returned if `start_positions` or `end_positions` is not provided):
Log probabilities for the `is_impossible` label of the answers.
mems (`List[torch.FloatTensor]` of length `config.n_layers`):
- Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding.
- The token ids which have their past given to this model should not be passed as `input_ids` as they
- have already been computed.
+ Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The
+ token ids which have their past given to this model should not be passed as `input_ids` as they have
+ already been computed.
hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
- Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer)
- of shape `(batch_size, sequence_length, hidden_size)`.
+ Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
+ shape `(batch_size, sequence_length, hidden_size)`.
Hidden-states of the model at the output of each layer plus the initial embedding outputs.
attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
- Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`.
+ Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+ sequence_length)`.
Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
heads.
@@ -837,19 +844,18 @@ class XLNetForQuestionAnsweringOutput(ModelOutput):
XLNET_START_DOCSTRING = r"""
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic
- methods the library implements for all its model (such as downloading or saving, resizing the input embeddings,
- pruning heads etc.)
+ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+ library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+ etc.)
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module)
- subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to
- general usage and behavior.
+ This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+ Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+ and behavior.
Parameters:
config ([`XLNetConfig`]): Model configuration class with all the parameters of the model.
Initializing with a config file does not load the weights associated with the model, only the
- configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model
- weights.
+ configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""
XLNET_INPUTS_DOCSTRING = r"""
@@ -857,9 +863,8 @@ XLNET_INPUTS_DOCSTRING = r"""
input_ids (`torch.LongTensor` of shape `({0})`):
Indices of input sequence tokens in the vocabulary.
- Indices can be obtained using [`XLNetTokenizer`]. See
- [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for
- details.
+ Indices can be obtained using [`XLNetTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+ [`PreTrainedTokenizer.__call__`] for details.
[What are input IDs?](../glossary#input-ids)
attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
@@ -871,8 +876,8 @@ XLNET_INPUTS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
mems (`List[torch.FloatTensor]` of length `config.n_layers`):
Contains pre-computed hidden-states (see `mems` output below) . Can be used to speed up sequential
- decoding. The token ids which have their past given to this model should not be passed as `input_ids`
- as they have already been computed.
+ decoding. The token ids which have their past given to this model should not be passed as `input_ids` as
+ they have already been computed.
`use_mems` has to be set to `True` to make use of `mems`.
perm_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length, sequence_length)`, *optional*):
@@ -884,19 +889,20 @@ XLNET_INPUTS_DOCSTRING = r"""
If not set, each token attends to all the others (full bidirectional attention). Only used during
pretraining (to define factorization order) or for sequential decoding (generation).
target_mapping (`torch.FloatTensor` of shape `(batch_size, num_predict, sequence_length)`, *optional*):
- Mask to indicate the output tokens to use. If `target_mapping[k, i, j] = 1`, the i-th predict in batch k
- is on the j-th token. Only used during pretraining for partial prediction or for sequential decoding
+ Mask to indicate the output tokens to use. If `target_mapping[k, i, j] = 1`, the i-th predict in batch k is
+ on the j-th token. Only used during pretraining for partial prediction or for sequential decoding
(generation).
token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
- Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`:
+ Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+ 1]`:
- 0 corresponds to a *sentence A* token,
- 1 corresponds to a *sentence B* token.
[What are token type IDs?](../glossary#token-type-ids)
input_mask (`torch.FloatTensor` of shape `{0}`, *optional*):
- Mask to avoid performing attention on padding token indices. Negative of `attention_mask`, i.e. with 0
- for real tokens and 1 for padding which is kept for compatibility with the original code base.
+ Mask to avoid performing attention on padding token indices. Negative of `attention_mask`, i.e. with 0 for
+ real tokens and 1 for padding which is kept for compatibility with the original code base.
Mask values selected in `[0, 1]`:
@@ -911,9 +917,9 @@ XLNET_INPUTS_DOCSTRING = r"""
- 0 indicates the head is **masked**.
inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation.
- This is useful if you want more control over how to convert `input_ids` indices into associated
- vectors than the model's internal embedding lookup matrix.
+ Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+ is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+ model's internal embedding lookup matrix.
output_attentions (`bool`, *optional*):
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
tensors for more detail.
@@ -969,13 +975,12 @@ class XLNetModel(XLNetPreTrainedModel):
::
- same_length=False: same_length=True:
- < qlen > < qlen >
- ^ [0 0 0 0 0 1 1 1 1] [0 0 0 0 0 1 1 1 1]
- [0 0 0 0 0 0 1 1 1] [1 0 0 0 0 0 1 1 1]
- qlen [0 0 0 0 0 0 0 1 1] [1 1 0 0 0 0 0 1 1]
- [0 0 0 0 0 0 0 0 1] [1 1 1 0 0 0 0 0 1]
- v [0 0 0 0 0 0 0 0 0] [1 1 1 1 0 0 0 0 0]
+ same_length=False: same_length=True: < qlen > < qlen >
+ ^ [0 0 0 0 0 1 1 1 1] [0 0 0 0 0 1 1 1 1]
+ [0 0 0 0 0 0 1 1 1] [1 0 0 0 0 0 1 1 1]
+ qlen [0 0 0 0 0 0 0 1 1] [1 1 0 0 0 0 0 1 1]
+ [0 0 0 0 0 0 0 0 1] [1 1 1 0 0 0 0 0 1]
+ v [0 0 0 0 0 0 0 0 0] [1 1 1 1 0 0 0 0 0]
"""
attn_mask = torch.ones([qlen, qlen])
@@ -1381,12 +1386,11 @@ class XLNetLMHeadModel(XLNetPreTrainedModel):
`target_mapping` is :obj*None*, then `num_predict` corresponds to `sequence_length`.
The labels should correspond to the masked input words that should be predicted and depends on
- `target_mapping`. Note in order to perform standard auto-regressive language modeling a ** token
- has to be added to the `input_ids` (see the `prepare_inputs_for_generation` function and examples
- below)
+ `target_mapping`. Note in order to perform standard auto-regressive language modeling a ** token has
+ to be added to the `input_ids` (see the `prepare_inputs_for_generation` function and examples below)
- Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` are ignored, the
- loss is only computed for labels in `[0, ..., config.vocab_size]`
+ Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` are ignored, the loss
+ is only computed for labels in `[0, ..., config.vocab_size]`
Return:
@@ -1465,8 +1469,8 @@ class XLNetLMHeadModel(XLNetPreTrainedModel):
def _reorder_cache(mems: List[torch.Tensor], beam_idx: torch.Tensor) -> List[torch.Tensor]:
"""
This function is used to re-order the `mems` cache if [`~PreTrainedModel.beam_search`] or
- [`~PreTrainedModel.beam_sample`] is called. This is required to match `mems` with the
- correct beam_idx at every generation step.
+ [`~PreTrainedModel.beam_sample`] is called. This is required to match `mems` with the correct beam_idx at every
+ generation step.
"""
return [layer_past.index_select(1, beam_idx.to(layer_past.device)) for layer_past in mems]
@@ -1518,8 +1522,9 @@ class XLNetForSequenceClassification(XLNetPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss),
- If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+ Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+ config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+ `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1625,8 +1630,8 @@ class XLNetForTokenClassification(XLNetPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices]` where *num_choices* is the size of the second dimension of the input tensors. (see
- *input_ids* above)
+ Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices]`
+ where *num_choices* is the size of the second dimension of the input tensors. (see *input_ids* above)
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1722,7 +1727,8 @@ class XLNetForMultipleChoice(XLNetPreTrainedModel):
):
r"""
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
- Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
+ Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
+ num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
`input_ids` above)
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1827,12 +1833,12 @@ class XLNetForQuestionAnsweringSimple(XLNetPreTrainedModel):
r"""
start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the start of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the end of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
"""
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
@@ -1939,12 +1945,12 @@ class XLNetForQuestionAnswering(XLNetPreTrainedModel):
r"""
start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the start of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels for position (index) of the end of the labelled span for computing the token classification loss.
- Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the
- sequence are not taken into account for computing the loss.
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
+ are not taken into account for computing the loss.
is_impossible (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
Labels whether a question has an answer or no answer (SQuAD 2.0)
cls_index (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
diff --git a/src/transformers/models/xlnet/tokenization_xlnet.py b/src/transformers/models/xlnet/tokenization_xlnet.py
index d84c568ce3..7f0c28d0c0 100644
--- a/src/transformers/models/xlnet/tokenization_xlnet.py
+++ b/src/transformers/models/xlnet/tokenization_xlnet.py
@@ -55,8 +55,8 @@ class XLNetTokenizer(PreTrainedTokenizer):
"""
Construct an XLNet tokenizer. Based on [SentencePiece](https://github.com/google/sentencepiece).
- This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods.
- Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
+ this superclass for more information regarding those methods.
Args:
vocab_file (`str`):
@@ -83,8 +83,8 @@ class XLNetTokenizer(PreTrainedTokenizer):
- When building a sequence using special tokens, this is not the token that is used for the end of
- sequence. The token used is the `sep_token`.
+ When building a sequence using special tokens, this is not the token that is used for the end of sequence.
+ The token used is the `sep_token`.
@@ -106,7 +106,9 @@ class XLNetTokenizer(PreTrainedTokenizer):
additional_special_tokens (`List[str]`, *optional*, defaults to `["", ""]`):
Additional special tokens used by the tokenizer.
sp_model_kwargs (`dict`, *optional*):
- Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things, to set:
+ Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for
+ SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things,
+ to set:
- `enable_sampling`: Enable subword regularization.
- `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout.
@@ -323,8 +325,7 @@ class XLNetTokenizer(PreTrainedTokenizer):
Optional second list of IDs for sequence pairs.
Returns:
- `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given
- sequence(s).
+ `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
"""
sep = [self.sep_token_id]
cls_segment_id = [2]
diff --git a/src/transformers/models/xlnet/tokenization_xlnet_fast.py b/src/transformers/models/xlnet/tokenization_xlnet_fast.py
index 1c7d938994..863f73a59b 100644
--- a/src/transformers/models/xlnet/tokenization_xlnet_fast.py
+++ b/src/transformers/models/xlnet/tokenization_xlnet_fast.py
@@ -63,10 +63,11 @@ SEG_ID_PAD = 4
class XLNetTokenizerFast(PreTrainedTokenizerFast):
"""
- Construct a "fast" XLNet tokenizer (backed by HuggingFace's *tokenizers* library). Based on [Unigram](https://huggingface.co/docs/tokenizers/python/latest/components.html?highlight=unigram#models).
+ Construct a "fast" XLNet tokenizer (backed by HuggingFace's *tokenizers* library). Based on
+ [Unigram](https://huggingface.co/docs/tokenizers/python/latest/components.html?highlight=unigram#models).
- This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main
- methods. Users should refer to this superclass for more information regarding those methods.
+ This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should
+ refer to this superclass for more information regarding those methods.
Args:
vocab_file (`str`):
@@ -93,8 +94,8 @@ class XLNetTokenizerFast(PreTrainedTokenizerFast):
- When building a sequence using special tokens, this is not the token that is used for the end of
- sequence. The token used is the `sep_token`.
+ When building a sequence using special tokens, this is not the token that is used for the end of sequence.
+ The token used is the `sep_token`.
@@ -217,8 +218,7 @@ class XLNetTokenizerFast(PreTrainedTokenizerFast):
Optional second list of IDs for sequence pairs.
Returns:
- `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given
- sequence(s).
+ `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
"""
sep = [self.sep_token_id]
cls_segment_id = [2]
diff --git a/src/transformers/onnx/convert.py b/src/transformers/onnx/convert.py
index 041e21832a..fed280d19e 100644
--- a/src/transformers/onnx/convert.py
+++ b/src/transformers/onnx/convert.py
@@ -224,9 +224,7 @@ def ensure_model_and_config_inputs_match(
) -> Tuple[bool, List[str]]:
"""
- :param model_inputs:
- :param config_inputs:
- :return:
+ :param model_inputs: :param config_inputs: :return:
"""
forward_parameters = signature(model.forward).parameters
model_inputs_set = set(model_inputs)
diff --git a/src/transformers/optimization.py b/src/transformers/optimization.py
index 5681a40710..f21a240ad6 100644
--- a/src/transformers/optimization.py
+++ b/src/transformers/optimization.py
@@ -271,7 +271,8 @@ def get_scheduler(
class AdamW(Optimizer):
"""
- Implements Adam algorithm with weight decay fix as introduced in [Decoupled Weight Decay Regularization](https://arxiv.org/abs/1711.05101).
+ Implements Adam algorithm with weight decay fix as introduced in [Decoupled Weight Decay
+ Regularization](https://arxiv.org/abs/1711.05101).
Parameters:
params (`Iterable[nn.parameter.Parameter]`):
@@ -427,7 +428,8 @@ class Adafactor(Optimizer):
Adafactor(model.parameters(), scale_parameter=True, relative_step=True, warmup_init=True, lr=None)
```
- When using `lr=None` with [`Trainer`] you will most likely need to use [`~optimization.AdafactorSchedule`] scheduler as following:
+ When using `lr=None` with [`Trainer`] you will most likely need to use [`~optimization.AdafactorSchedule`]
+ scheduler as following:
```python
from transformers.optimization import Adafactor, AdafactorSchedule
@@ -611,9 +613,8 @@ class Adafactor(Optimizer):
class AdafactorSchedule(LambdaLR):
"""
- Since [`~optimization.Adafactor`] performs its own scheduling, if the training loop relies on a
- scheduler (e.g., for logging), this class creates a proxy object that retrieves the current lr values from the
- optimizer.
+ Since [`~optimization.Adafactor`] performs its own scheduling, if the training loop relies on a scheduler (e.g.,
+ for logging), this class creates a proxy object that retrieves the current lr values from the optimizer.
It returns `initial_lr` during startup and the actual `lr` during stepping.
"""
diff --git a/src/transformers/optimization_tf.py b/src/transformers/optimization_tf.py
index d18f85fe78..02acb8aa5e 100644
--- a/src/transformers/optimization_tf.py
+++ b/src/transformers/optimization_tf.py
@@ -153,7 +153,8 @@ class AdamWeightDecay(tf.keras.optimizers.Adam):
"""
Adam enables L2 weight decay and clip_by_global_norm on gradients. Just adding the square of the weights to the
loss function is *not* the correct way of using L2 regularization/weight decay with Adam, since that will interact
- with the m and v parameters in strange ways as shown in [Decoupled Weight Decay Regularization](https://arxiv.org/abs/1711.05101).
+ with the m and v parameters in strange ways as shown in [Decoupled Weight Decay
+ Regularization](https://arxiv.org/abs/1711.05101).
Instead we want ot decay the weights in a manner that doesn't interact with the m/v parameters. This is equivalent
to adding the square of the weights to the loss with plain (non-momentum) SGD.
@@ -168,7 +169,8 @@ class AdamWeightDecay(tf.keras.optimizers.Adam):
epsilon (`float`, *optional*, defaults to 1e-7):
The epsilon parameter in Adam, which is a small constant for numerical stability.
amsgrad (`bool`, *optional*, default to *False*):
- Whether to apply AMSGrad variant of this algorithm or not, see [On the Convergence of Adam and Beyond](https://arxiv.org/abs/1904.09237).
+ Whether to apply AMSGrad variant of this algorithm or not, see [On the Convergence of Adam and
+ Beyond](https://arxiv.org/abs/1904.09237).
weight_decay_rate (`float`, *optional*, defaults to 0):
The weight decay to apply.
include_in_weight_decay (`List[str]`, *optional*):
@@ -180,10 +182,10 @@ class AdamWeightDecay(tf.keras.optimizers.Adam):
name (`str`, *optional*, defaults to 'AdamWeightDecay'):
Optional name for the operations created when applying gradients.
kwargs:
- Keyword arguments. Allowed to be {`clipnorm`, `clipvalue`, `lr`, `decay`}. `clipnorm` is clip
- gradients by norm; `clipvalue` is clip gradients by value, `decay` is included for backward
- compatibility to allow time inverse decay of learning rate. `lr` is included for backward compatibility,
- recommended to use `learning_rate` instead.
+ Keyword arguments. Allowed to be {`clipnorm`, `clipvalue`, `lr`, `decay`}. `clipnorm` is clip gradients by
+ norm; `clipvalue` is clip gradients by value, `decay` is included for backward compatibility to allow time
+ inverse decay of learning rate. `lr` is included for backward compatibility, recommended to use
+ `learning_rate` instead.
"""
def __init__(
diff --git a/src/transformers/pipelines/__init__.py b/src/transformers/pipelines/__init__.py
index 98c59ae25f..229963175f 100755
--- a/src/transformers/pipelines/__init__.py
+++ b/src/transformers/pipelines/__init__.py
@@ -338,8 +338,8 @@ def check_task(task: str) -> Tuple[Dict, Any]:
- `"zero-shot-classification"`
Returns:
- (task_defaults`dict`, task_options: (`tuple`, None)) The actual dictionary required to initialize the
- pipeline and some extra task options for parametrized tasks like "translation_XX_to_YY"
+ (task_defaults`dict`, task_options: (`tuple`, None)) The actual dictionary required to initialize the pipeline
+ and some extra task options for parametrized tasks like "translation_XX_to_YY"
"""
@@ -387,8 +387,7 @@ def pipeline(
The task defining which pipeline will be returned. Currently accepted tasks are:
- `"audio-classification"`: will return a [`AudioClassificationPipeline`].
- - `"automatic-speech-recognition"`: will return a
- [`AutomaticSpeechRecognitionPipeline`].
+ - `"automatic-speech-recognition"`: will return a [`AutomaticSpeechRecognitionPipeline`].
- `"conversational"`: will return a [`ConversationalPipeline`].
- `"feature-extraction"`: will return a [`FeatureExtractionPipeline`].
- `"fill-mask"`: will return a [`FillMaskPipeline`]:.
@@ -399,8 +398,7 @@ def pipeline(
- `"text-classification"` (alias `"sentiment-analysis"` available): will return a
[`TextClassificationPipeline`].
- `"text-generation"`: will return a [`TextGenerationPipeline`]:.
- - `"token-classification"` (alias `"ner"` available): will return a
- [`TokenClassificationPipeline`].
+ - `"token-classification"` (alias `"ner"` available): will return a [`TokenClassificationPipeline`].
- `"translation"`: will return a [`TranslationPipeline`].
- `"translation_xx_to_yy"`: will return a [`TranslationPipeline`].
- `"summarization"`: will return a [`SummarizationPipeline`].
@@ -408,45 +406,43 @@ def pipeline(
model (`str` or [`PreTrainedModel`] or [`TFPreTrainedModel`], *optional*):
The model that will be used by the pipeline to make predictions. This can be a model identifier or an
- actual instance of a pretrained model inheriting from [`PreTrainedModel`] (for PyTorch)
- or [`TFPreTrainedModel`] (for TensorFlow).
+ actual instance of a pretrained model inheriting from [`PreTrainedModel`] (for PyTorch) or
+ [`TFPreTrainedModel`] (for TensorFlow).
If not provided, the default for the `task` will be loaded.
config (`str` or [`PretrainedConfig`], *optional*):
The configuration that will be used by the pipeline to instantiate the model. This can be a model
- identifier or an actual pretrained model configuration inheriting from
- [`PretrainedConfig`].
+ identifier or an actual pretrained model configuration inheriting from [`PretrainedConfig`].
If not provided, the default configuration file for the requested model will be used. That means that if
- `model` is given, its default configuration will be used. However, if `model` is not supplied,
- this `task`'s default model's config is used instead.
+ `model` is given, its default configuration will be used. However, if `model` is not supplied, this
+ `task`'s default model's config is used instead.
tokenizer (`str` or [`PreTrainedTokenizer`], *optional*):
The tokenizer that will be used by the pipeline to encode data for the model. This can be a model
identifier or an actual pretrained tokenizer inheriting from [`PreTrainedTokenizer`].
- If not provided, the default tokenizer for the given `model` will be loaded (if it is a string). If
- `model` is not specified or not a string, then the default tokenizer for `config` is loaded (if
- it is a string). However, if `config` is also not given or not a string, then the default tokenizer
- for the given `task` will be loaded.
+ If not provided, the default tokenizer for the given `model` will be loaded (if it is a string). If `model`
+ is not specified or not a string, then the default tokenizer for `config` is loaded (if it is a string).
+ However, if `config` is also not given or not a string, then the default tokenizer for the given `task`
+ will be loaded.
feature_extractor (`str` or [`PreTrainedFeatureExtractor`], *optional*):
The feature extractor that will be used by the pipeline to encode data for the model. This can be a model
- identifier or an actual pretrained feature extractor inheriting from
- [`PreTrainedFeatureExtractor`].
+ identifier or an actual pretrained feature extractor inheriting from [`PreTrainedFeatureExtractor`].
Feature extractors are used for non-NLP models, such as Speech or Vision models as well as multi-modal
models. Multi-modal models will also require a tokenizer to be passed.
- If not provided, the default feature extractor for the given `model` will be loaded (if it is a
- string). If `model` is not specified or not a string, then the default feature extractor for
- `config` is loaded (if it is a string). However, if `config` is also not given or not a string,
- then the default feature extractor for the given `task` will be loaded.
+ If not provided, the default feature extractor for the given `model` will be loaded (if it is a string). If
+ `model` is not specified or not a string, then the default feature extractor for `config` is loaded (if it
+ is a string). However, if `config` is also not given or not a string, then the default feature extractor
+ for the given `task` will be loaded.
framework (`str`, *optional*):
- The framework to use, either `"pt"` for PyTorch or `"tf"` for TensorFlow. The specified framework
- must be installed.
+ The framework to use, either `"pt"` for PyTorch or `"tf"` for TensorFlow. The specified framework must be
+ installed.
If no framework is specified, will default to the one currently installed. If no framework is specified and
- both frameworks are installed, will default to the framework of the `model`, or to PyTorch if no model
- is provided.
+ both frameworks are installed, will default to the framework of the `model`, or to PyTorch if no model is
+ provided.
revision(`str`, *optional*, defaults to `"main"`):
When passing a task name or a string model identifier: The specific model version to use. It can be a
branch name, a tag name, or a commit id, since we use a git-based system for storing models and other
@@ -454,11 +450,12 @@ def pipeline(
use_fast (`bool`, *optional*, defaults to `True`):
Whether or not to use a Fast tokenizer if possible (a [`PreTrainedTokenizerFast`]).
use_auth_token (`str` or *bool*, *optional*):
- The token to use as HTTP bearer authorization for remote files. If `True`, will use the token
- generated when running `transformers-cli login` (stored in `~/.huggingface`).
- revision(`str`, *optional*, defaults to `"main"`):
+ The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated
+ when running `transformers-cli login` (stored in `~/.huggingface`). revision(`str`, *optional*, defaults to
+ `"main"`):
model_kwargs:
- Additional dictionary of keyword arguments passed along to the model's `from_pretrained(..., **model_kwargs)` function.
+ Additional dictionary of keyword arguments passed along to the model's `from_pretrained(...,
+ **model_kwargs)` function.
kwargs:
Additional keyword arguments passed along to the specific pipeline init (see the documentation for the
corresponding pipeline class for possible values).
diff --git a/src/transformers/pipelines/audio_classification.py b/src/transformers/pipelines/audio_classification.py
index 882d01f77e..6f86cb803c 100644
--- a/src/transformers/pipelines/audio_classification.py
+++ b/src/transformers/pipelines/audio_classification.py
@@ -66,14 +66,15 @@ def ffmpeg_read(bpayload: bytes, sampling_rate: int) -> np.array:
@add_end_docstrings(PIPELINE_INIT_ARGS)
class AudioClassificationPipeline(Pipeline):
"""
- Audio classification pipeline using any `AutoModelForAudioClassification`. This pipeline predicts the class of
- a raw waveform or an audio file. In case of an audio file, ffmpeg should be installed to support multiple audio
+ Audio classification pipeline using any `AutoModelForAudioClassification`. This pipeline predicts the class of a
+ raw waveform or an audio file. In case of an audio file, ffmpeg should be installed to support multiple audio
formats.
This pipeline can currently be loaded from [`pipeline`] using the following task identifier:
`"audio-classification"`.
- See the list of available models on [huggingface.co/models](https://huggingface.co/models?filter=audio-classification).
+ See the list of available models on
+ [huggingface.co/models](https://huggingface.co/models?filter=audio-classification).
"""
def __init__(self, *args, **kwargs):
@@ -92,16 +93,16 @@ class AudioClassificationPipeline(Pipeline):
**kwargs,
):
"""
- Classify the sequence(s) given as inputs. See the [`AutomaticSpeechRecognitionPipeline`]
- documentation for more information.
+ Classify the sequence(s) given as inputs. See the [`AutomaticSpeechRecognitionPipeline`] documentation for more
+ information.
Args:
inputs (`np.ndarray` or `bytes` or `str`):
- The inputs is either a raw waveform (`np.ndarray` of shape (n, ) of type `np.float32` or
- `np.float64`) at the correct sampling rate (no further check will be done) or a `str` that is
- the filename of the audio file, the file will be read at the correct sampling rate to get the waveform
- using *ffmpeg*. This requires *ffmpeg* to be installed on the system. If *inputs* is `bytes` it is
- supposed to be the content of an audio file and is interpreted by *ffmpeg* in the same way.
+ The inputs is either a raw waveform (`np.ndarray` of shape (n, ) of type `np.float32` or `np.float64`)
+ at the correct sampling rate (no further check will be done) or a `str` that is the filename of the
+ audio file, the file will be read at the correct sampling rate to get the waveform using *ffmpeg*. This
+ requires *ffmpeg* to be installed on the system. If *inputs* is `bytes` it is supposed to be the
+ content of an audio file and is interpreted by *ffmpeg* in the same way.
top_k (`int`, *optional*, defaults to None):
The number of top labels that will be returned by the pipeline. If the provided number is *None* or
higher than the number of labels available in the model configuration, it will default to the number of
diff --git a/src/transformers/pipelines/automatic_speech_recognition.py b/src/transformers/pipelines/automatic_speech_recognition.py
index a6b2ec6e69..d01fd90e28 100644
--- a/src/transformers/pipelines/automatic_speech_recognition.py
+++ b/src/transformers/pipelines/automatic_speech_recognition.py
@@ -81,20 +81,19 @@ class AutomaticSpeechRecognitionPipeline(Pipeline):
The feature extractor that will be used by the pipeline to encode waveform for the model.
model ([`PreTrainedModel`] or [`TFPreTrainedModel`]):
The model that will be used by the pipeline to make predictions. This needs to be a model inheriting
- from [`PreTrainedModel`] for PyTorch and [`TFPreTrainedModel`]
- for TensorFlow.
+ from [`PreTrainedModel`] for PyTorch and [`TFPreTrainedModel`] for TensorFlow.
tokenizer ([`PreTrainedTokenizer`]):
The tokenizer that will be used by the pipeline to encode data for the model. This object inherits from
[`PreTrainedTokenizer`].
modelcard (`str` or [`ModelCard`], *optional*):
Model card attributed to the model for this pipeline.
framework (`str`, *optional*):
- The framework to use, either `"pt"` for PyTorch or `"tf"` for TensorFlow. The specified
- framework must be installed.
+ The framework to use, either `"pt"` for PyTorch or `"tf"` for TensorFlow. The specified framework must
+ be installed.
If no framework is specified, will default to the one currently installed. If no framework is specified
- and both frameworks are installed, will default to the framework of the `model`, or to PyTorch if
- no model is provided.
+ and both frameworks are installed, will default to the framework of the `model`, or to PyTorch if no
+ model is provided.
device (`int`, *optional*, defaults to -1):
Device ordinal for CPU/GPU supports. Setting this to -1 will leverage CPU, a positive will run the
model on the associated CUDA device id.
@@ -114,16 +113,16 @@ class AutomaticSpeechRecognitionPipeline(Pipeline):
**kwargs,
):
"""
- Classify the sequence(s) given as inputs. See the [`AutomaticSpeechRecognitionPipeline`]
- documentation for more information.
+ Classify the sequence(s) given as inputs. See the [`AutomaticSpeechRecognitionPipeline`] documentation for more
+ information.
Args:
inputs (`np.ndarray` or `bytes` or `str`):
- The inputs is either a raw waveform (`np.ndarray` of shape (n, ) of type `np.float32` or
- `np.float64`) at the correct sampling rate (no further check will be done) or a `str` that is
- the filename of the audio file, the file will be read at the correct sampling rate to get the waveform
- using *ffmpeg*. This requires *ffmpeg* to be installed on the system. If *inputs* is `bytes` it is
- supposed to be the content of an audio file and is interpreted by *ffmpeg* in the same way.
+ The inputs is either a raw waveform (`np.ndarray` of shape (n, ) of type `np.float32` or `np.float64`)
+ at the correct sampling rate (no further check will be done) or a `str` that is the filename of the
+ audio file, the file will be read at the correct sampling rate to get the waveform using *ffmpeg*. This
+ requires *ffmpeg* to be installed on the system. If *inputs* is `bytes` it is supposed to be the
+ content of an audio file and is interpreted by *ffmpeg* in the same way.
Return:
A `dict` with the following keys:
diff --git a/src/transformers/pipelines/base.py b/src/transformers/pipelines/base.py
index f2608a596d..fe98c36530 100644
--- a/src/transformers/pipelines/base.py
+++ b/src/transformers/pipelines/base.py
@@ -152,16 +152,15 @@ def infer_framework_load_model(
"""
Select framework (TensorFlow or PyTorch) to use from the `model` passed. Returns a tuple (framework, model).
- If `model` is instantiated, this function will just infer the framework from the model class. Otherwise
- `model` is actually a checkpoint name and this method will try to instantiate it using `model_classes`.
- Since we don't want to instantiate the model twice, this model is returned for use by the pipeline.
+ If `model` is instantiated, this function will just infer the framework from the model class. Otherwise `model` is
+ actually a checkpoint name and this method will try to instantiate it using `model_classes`. Since we don't want to
+ instantiate the model twice, this model is returned for use by the pipeline.
If both frameworks are installed and available for `model`, PyTorch is selected.
Args:
model (`str`, [`PreTrainedModel`] or [`TFPreTrainedModel`]):
- The model to infer the framework from. If `str`, a checkpoint name. The model to infer the framewrok
- from.
+ The model to infer the framework from. If `str`, a checkpoint name. The model to infer the framewrok from.
config ([`AutoConfig`]):
The config associated with the model to help using the correct class
model_classes (dictionary `str` to `type`, *optional*):
@@ -169,7 +168,8 @@ def infer_framework_load_model(
task (`str`):
The task defining which pipeline will be returned.
model_kwargs:
- Additional dictionary of keyword arguments passed along to the model's `from_pretrained(..., **model_kwargs)` function.
+ Additional dictionary of keyword arguments passed along to the model's `from_pretrained(...,
+ **model_kwargs)` function.
Returns:
`Tuple`: A tuple framework, model.
@@ -248,22 +248,22 @@ def infer_framework_from_model(
"""
Select framework (TensorFlow or PyTorch) to use from the `model` passed. Returns a tuple (framework, model).
- If `model` is instantiated, this function will just infer the framework from the model class. Otherwise
- `model` is actually a checkpoint name and this method will try to instantiate it using `model_classes`.
- Since we don't want to instantiate the model twice, this model is returned for use by the pipeline.
+ If `model` is instantiated, this function will just infer the framework from the model class. Otherwise `model` is
+ actually a checkpoint name and this method will try to instantiate it using `model_classes`. Since we don't want to
+ instantiate the model twice, this model is returned for use by the pipeline.
If both frameworks are installed and available for `model`, PyTorch is selected.
Args:
model (`str`, [`PreTrainedModel`] or [`TFPreTrainedModel`]):
- The model to infer the framework from. If `str`, a checkpoint name. The model to infer the framewrok
- from.
+ The model to infer the framework from. If `str`, a checkpoint name. The model to infer the framewrok from.
model_classes (dictionary `str` to `type`, *optional*):
A mapping framework to class.
task (`str`):
The task defining which pipeline will be returned.
model_kwargs:
- Additional dictionary of keyword arguments passed along to the model's `from_pretrained(..., **model_kwargs)` function.
+ Additional dictionary of keyword arguments passed along to the model's `from_pretrained(...,
+ **model_kwargs)` function.
Returns:
`Tuple`: A tuple framework, model.
@@ -389,8 +389,8 @@ class PipelineDataFormat:
- CSV
- stdin/stdout (pipe)
- `PipelineDataFormat` also includes some utilities to work with multi-columns like mapping from datasets
- columns to pipelines keyword arguments through the `dataset_kwarg_1=dataset_column_1` format.
+ `PipelineDataFormat` also includes some utilities to work with multi-columns like mapping from datasets columns to
+ pipelines keyword arguments through the `dataset_kwarg_1=dataset_column_1` format.
Args:
output_path (`str`, *optional*): Where to save the outgoing data.
@@ -432,8 +432,7 @@ class PipelineDataFormat:
@abstractmethod
def save(self, data: Union[dict, List[dict]]):
"""
- Save the provided data object with the representation for the current
- [`~pipelines.PipelineDataFormat`].
+ Save the provided data object with the representation for the current [`~pipelines.PipelineDataFormat`].
Args:
data (`dict` or list of `dict`): The data to store.
@@ -467,8 +466,7 @@ class PipelineDataFormat:
overwrite=False,
) -> "PipelineDataFormat":
"""
- Creates an instance of the right subclass of [`~pipelines.PipelineDataFormat`] depending on
- `format`.
+ Creates an instance of the right subclass of [`~pipelines.PipelineDataFormat`] depending on `format`.
Args:
format: (`str`):
@@ -527,8 +525,7 @@ class CsvPipelineDataFormat(PipelineDataFormat):
def save(self, data: List[dict]):
"""
- Save the provided data object with the representation for the current
- [`~pipelines.PipelineDataFormat`].
+ Save the provided data object with the representation for the current [`~pipelines.PipelineDataFormat`].
Args:
data (`List[dict]`): The data to store.
@@ -649,20 +646,19 @@ PIPELINE_INIT_ARGS = r"""
Arguments:
model ([`PreTrainedModel`] or [`TFPreTrainedModel`]):
The model that will be used by the pipeline to make predictions. This needs to be a model inheriting from
- [`PreTrainedModel`] for PyTorch and [`TFPreTrainedModel`] for
- TensorFlow.
+ [`PreTrainedModel`] for PyTorch and [`TFPreTrainedModel`] for TensorFlow.
tokenizer ([`PreTrainedTokenizer`]):
The tokenizer that will be used by the pipeline to encode data for the model. This object inherits from
[`PreTrainedTokenizer`].
modelcard (`str` or [`ModelCard`], *optional*):
Model card attributed to the model for this pipeline.
framework (`str`, *optional*):
- The framework to use, either `"pt"` for PyTorch or `"tf"` for TensorFlow. The specified framework
- must be installed.
+ The framework to use, either `"pt"` for PyTorch or `"tf"` for TensorFlow. The specified framework must be
+ installed.
If no framework is specified, will default to the one currently installed. If no framework is specified and
- both frameworks are installed, will default to the framework of the `model`, or to PyTorch if no model
- is provided.
+ both frameworks are installed, will default to the framework of the `model`, or to PyTorch if no model is
+ provided.
task (`str`, defaults to `""`):
A task-identifier for the pipeline.
num_workers (`int`, *optional*, defaults to 8):
@@ -670,7 +666,8 @@ PIPELINE_INIT_ARGS = r"""
workers to be used.
batch_size (`int`, *optional*, defaults to 1):
When the pipeline will use *DataLoader* (when passing a dataset, on GPU for a Pytorch model), the size of
- the batch to use, for inference this is not always beneficial, please read [Batching with pipelines](https://huggingface.co/transformers/main_classes/pipelines.html#pipeline-batching) .
+ the batch to use, for inference this is not always beneficial, please read [Batching with
+ pipelines](https://huggingface.co/transformers/main_classes/pipelines.html#pipeline-batching) .
args_parser ([`~pipelines.ArgumentHandler`], *optional*):
Reference to the object in charge of parsing supplied pipeline parameters.
device (`int`, *optional*, defaults to -1):
@@ -702,10 +699,9 @@ class Pipeline(_ScikitCompat):
Pipeline supports running on CPU or GPU through the device argument (see below).
- Some pipeline, like for instance [`FeatureExtractionPipeline`] (`'feature-extraction'`)
- output large tensor object as nested-lists. In order to avoid dumping such large structure as textual data we
- provide the `binary_output` constructor argument. If set to `True`, the output will be stored in the
- pickle format.
+ Some pipeline, like for instance [`FeatureExtractionPipeline`] (`'feature-extraction'`) output large tensor object
+ as nested-lists. In order to avoid dumping such large structure as textual data we provide the `binary_output`
+ constructor argument. If set to `True`, the output will be stored in the pickle format.
"""
default_input_names = None
@@ -815,7 +811,8 @@ class Pipeline(_ScikitCompat):
Ensure PyTorch tensors are on the specified device.
Args:
- inputs (keyword arguments that should be `torch.Tensor`, the rest is ignored): The tensors to place on `self.device`.
+ inputs (keyword arguments that should be `torch.Tensor`, the rest is ignored):
+ The tensors to place on `self.device`.
Recursive on lists **only**.
Return:
diff --git a/src/transformers/pipelines/conversational.py b/src/transformers/pipelines/conversational.py
index c5653e57e4..ab28bc2e65 100644
--- a/src/transformers/pipelines/conversational.py
+++ b/src/transformers/pipelines/conversational.py
@@ -19,17 +19,15 @@ logger = logging.get_logger(__name__)
class Conversation:
"""
Utility class containing a conversation and its history. This class is meant to be used as an input to the
- [`ConversationalPipeline`]. The conversation contains a number of utility function to manage the
- addition of new user input and generated model responses. A conversation needs to contain an unprocessed user input
- before being passed to the [`ConversationalPipeline`]. This user input is either created when
- the class is instantiated, or by calling `conversational_pipeline.append_response("input")` after a
- conversation turn.
+ [`ConversationalPipeline`]. The conversation contains a number of utility function to manage the addition of new
+ user input and generated model responses. A conversation needs to contain an unprocessed user input before being
+ passed to the [`ConversationalPipeline`]. This user input is either created when the class is instantiated, or by
+ calling `conversational_pipeline.append_response("input")` after a conversation turn.
Arguments:
text (`str`, *optional*):
The initial user input to start the conversation. If not provided, a user input needs to be provided
- manually using the [`~Conversation.add_user_input`] method before the conversation can
- begin.
+ manually using the [`~Conversation.add_user_input`] method before the conversation can begin.
conversation_id (`uuid.UUID`, *optional*):
Unique identifier for the conversation. If not provided, a random UUID4 id will be assigned to the
conversation.
@@ -84,8 +82,7 @@ class Conversation:
def add_user_input(self, text: str, overwrite: bool = False):
"""
- Add a user input to the conversation for the next round. This populates the internal `new_user_input`
- field.
+ Add a user input to the conversation for the next round. This populates the internal `new_user_input` field.
Args:
text (`str`): The user input for the next conversation round.
@@ -109,8 +106,8 @@ class Conversation:
def mark_processed(self):
"""
- Mark the conversation as processed (moves the content of `new_user_input` to `past_user_inputs`) and
- empties the `new_user_input` field.
+ Mark the conversation as processed (moves the content of `new_user_input` to `past_user_inputs`) and empties
+ the `new_user_input` field.
"""
if self.new_user_input:
self.past_user_inputs.append(self.new_user_input)
@@ -129,8 +126,8 @@ class Conversation:
"""
Iterates over all blobs of the conversation.
- Returns: Iterator of (is_user, text_chunk) in chronological order of the conversation. `is_user` is a
- `bool`, `text_chunks` is a `str`.
+ Returns: Iterator of (is_user, text_chunk) in chronological order of the conversation. `is_user` is a `bool`,
+ `text_chunks` is a `str`.
"""
for user_input, generated_response in zip(self.past_user_inputs, self.generated_responses):
yield True, user_input
@@ -168,12 +165,13 @@ class ConversationalPipeline(Pipeline):
"""
Multi-turn conversational pipeline.
- This conversational pipeline can currently be loaded from [`pipeline`] using the following task
- identifier: `"conversational"`.
+ This conversational pipeline can currently be loaded from [`pipeline`] using the following task identifier:
+ `"conversational"`.
The models that this pipeline can use are models that have been fine-tuned on a multi-turn conversational task,
currently: *'microsoft/DialoGPT-small'*, *'microsoft/DialoGPT-medium'*, *'microsoft/DialoGPT-large'*. See the
- up-to-date list of available models on [huggingface.co/models](https://huggingface.co/models?filter=conversational).
+ up-to-date list of available models on
+ [huggingface.co/models](https://huggingface.co/models?filter=conversational).
Usage:
@@ -232,8 +230,8 @@ class ConversationalPipeline(Pipeline):
corresponding to your framework [here](./model#generative-models)).
Returns:
- [`Conversation`] or a list of [`Conversation`]: Conversation(s) with
- updated generated responses for those containing a new user input.
+ [`Conversation`] or a list of [`Conversation`]: Conversation(s) with updated generated responses for those
+ containing a new user input.
"""
# XXX: num_workers==0 is required to be backward compatible
# Otherwise the threads will require a Conversation copy.
diff --git a/src/transformers/pipelines/feature_extraction.py b/src/transformers/pipelines/feature_extraction.py
index 082a5f4027..5c9e316cc6 100644
--- a/src/transformers/pipelines/feature_extraction.py
+++ b/src/transformers/pipelines/feature_extraction.py
@@ -9,8 +9,8 @@ class FeatureExtractionPipeline(Pipeline):
Feature extraction pipeline using no model head. This pipeline extracts the hidden states from the base
transformer, which can be used as features in downstream tasks.
- This feature extraction pipeline can currently be loaded from [`pipeline`] using the task
- identifier: `"feature-extraction"`.
+ This feature extraction pipeline can currently be loaded from [`pipeline`] using the task identifier:
+ `"feature-extraction"`.
All models may be used for this pipeline. See a list of all models, including community-contributed models on
[huggingface.co/models](https://huggingface.co/models).
@@ -18,20 +18,19 @@ class FeatureExtractionPipeline(Pipeline):
Arguments:
model ([`PreTrainedModel`] or [`TFPreTrainedModel`]):
The model that will be used by the pipeline to make predictions. This needs to be a model inheriting from
- [`PreTrainedModel`] for PyTorch and [`TFPreTrainedModel`] for
- TensorFlow.
+ [`PreTrainedModel`] for PyTorch and [`TFPreTrainedModel`] for TensorFlow.
tokenizer ([`PreTrainedTokenizer`]):
The tokenizer that will be used by the pipeline to encode data for the model. This object inherits from
[`PreTrainedTokenizer`].
modelcard (`str` or [`ModelCard`], *optional*):
Model card attributed to the model for this pipeline.
framework (`str`, *optional*):
- The framework to use, either `"pt"` for PyTorch or `"tf"` for TensorFlow. The specified framework
- must be installed.
+ The framework to use, either `"pt"` for PyTorch or `"tf"` for TensorFlow. The specified framework must be
+ installed.
If no framework is specified, will default to the one currently installed. If no framework is specified and
- both frameworks are installed, will default to the framework of the `model`, or to PyTorch if no model
- is provided.
+ both frameworks are installed, will default to the framework of the `model`, or to PyTorch if no model is
+ provided.
task (`str`, defaults to `""`):
A task-identifier for the pipeline.
args_parser ([`~pipelines.ArgumentHandler`], *optional*):
diff --git a/src/transformers/pipelines/fill_mask.py b/src/transformers/pipelines/fill_mask.py
index 7247cd0477..4a81d66fd4 100644
--- a/src/transformers/pipelines/fill_mask.py
+++ b/src/transformers/pipelines/fill_mask.py
@@ -35,8 +35,8 @@ class FillMaskPipeline(Pipeline):
Masked language modeling prediction pipeline using any `ModelWithLMHead`. See the [masked language modeling
examples](../task_summary#masked-language-modeling) for more information.
- This mask filling pipeline can currently be loaded from [`pipeline`] using the following task
- identifier: `"fill-mask"`.
+ This mask filling pipeline can currently be loaded from [`pipeline`] using the following task identifier:
+ `"fill-mask"`.
The models that this pipeline can use are models that have been trained with a masked language modeling objective,
which includes the bi-directional models in the library. See the up-to-date list of available models on
@@ -45,8 +45,8 @@ class FillMaskPipeline(Pipeline):
This pipeline only works for inputs with exactly one token masked. Experimental: We added support for multiple
- masks. The returned values are raw model output, and correspond to disjoint probabilities where one might
- expect joint probabilities (See [discussion](https://github.com/huggingface/transformers/pull/10222)).
+ masks. The returned values are raw model output, and correspond to disjoint probabilities where one might expect
+ joint probabilities (See [discussion](https://github.com/huggingface/transformers/pull/10222)).
"""
diff --git a/src/transformers/pipelines/image_classification.py b/src/transformers/pipelines/image_classification.py
index 466566f37c..590a823911 100644
--- a/src/transformers/pipelines/image_classification.py
+++ b/src/transformers/pipelines/image_classification.py
@@ -19,13 +19,14 @@ logger = logging.get_logger(__name__)
@add_end_docstrings(PIPELINE_INIT_ARGS)
class ImageClassificationPipeline(Pipeline):
"""
- Image classification pipeline using any `AutoModelForImageClassification`. This pipeline predicts the class of
- an image.
+ Image classification pipeline using any `AutoModelForImageClassification`. This pipeline predicts the class of an
+ image.
- This image classification pipeline can currently be loaded from [`pipeline`] using the following
- task identifier: `"image-classification"`.
+ This image classification pipeline can currently be loaded from [`pipeline`] using the following task identifier:
+ `"image-classification"`.
- See the list of available models on [huggingface.co/models](https://huggingface.co/models?filter=image-classification).
+ See the list of available models on
+ [huggingface.co/models](https://huggingface.co/models?filter=image-classification).
"""
def __init__(self, *args, **kwargs):
diff --git a/src/transformers/pipelines/image_segmentation.py b/src/transformers/pipelines/image_segmentation.py
index 4effb11290..4306f035ef 100644
--- a/src/transformers/pipelines/image_segmentation.py
+++ b/src/transformers/pipelines/image_segmentation.py
@@ -29,13 +29,14 @@ Predictions = List[Prediction]
@add_end_docstrings(PIPELINE_INIT_ARGS)
class ImageSegmentationPipeline(Pipeline):
"""
- Image segmentation pipeline using any `AutoModelForImageSegmentation`. This pipeline predicts masks of objects
- and their classes.
+ Image segmentation pipeline using any `AutoModelForImageSegmentation`. This pipeline predicts masks of objects and
+ their classes.
- This image segmntation pipeline can currently be loaded from [`pipeline`] using the following
- task identifier: `"image-segmentation"`.
+ This image segmntation pipeline can currently be loaded from [`pipeline`] using the following task identifier:
+ `"image-segmentation"`.
- See the list of available models on [huggingface.co/models](https://huggingface.co/models?filter=image-segmentation).
+ See the list of available models on
+ [huggingface.co/models](https://huggingface.co/models?filter=image-segmentation).
"""
def __init__(self, *args, **kwargs):
diff --git a/src/transformers/pipelines/object_detection.py b/src/transformers/pipelines/object_detection.py
index 4cfcf8435c..2d801ee252 100644
--- a/src/transformers/pipelines/object_detection.py
+++ b/src/transformers/pipelines/object_detection.py
@@ -24,11 +24,11 @@ Predictions = List[Prediction]
@add_end_docstrings(PIPELINE_INIT_ARGS)
class ObjectDetectionPipeline(Pipeline):
"""
- Object detection pipeline using any `AutoModelForObjectDetection`. This pipeline predicts bounding boxes of
- objects and their classes.
+ Object detection pipeline using any `AutoModelForObjectDetection`. This pipeline predicts bounding boxes of objects
+ and their classes.
- This object detection pipeline can currently be loaded from [`pipeline`] using the following task
- identifier: `"object-detection"`.
+ This object detection pipeline can currently be loaded from [`pipeline`] using the following task identifier:
+ `"object-detection"`.
See the list of available models on [huggingface.co/models](https://huggingface.co/models?filter=object-detection).
"""
diff --git a/src/transformers/pipelines/question_answering.py b/src/transformers/pipelines/question_answering.py
index 73d98ed197..48d977d71e 100644
--- a/src/transformers/pipelines/question_answering.py
+++ b/src/transformers/pipelines/question_answering.py
@@ -34,8 +34,8 @@ class QuestionAnsweringArgumentHandler(ArgumentHandler):
QuestionAnsweringPipeline requires the user to provide multiple arguments (i.e. question & context) to be mapped to
internal [`SquadExample`].
- QuestionAnsweringArgumentHandler manages all the possible to create a [`SquadExample`] from the
- command-line supplied arguments.
+ QuestionAnsweringArgumentHandler manages all the possible to create a [`SquadExample`] from the command-line
+ supplied arguments.
"""
def normalize(self, item):
@@ -101,13 +101,15 @@ class QuestionAnsweringArgumentHandler(ArgumentHandler):
@add_end_docstrings(PIPELINE_INIT_ARGS)
class QuestionAnsweringPipeline(ChunkPipeline):
"""
- Question Answering pipeline using any `ModelForQuestionAnswering`. See the [question answering examples](../task_summary#question-answering) for more information.
+ Question Answering pipeline using any `ModelForQuestionAnswering`. See the [question answering
+ examples](../task_summary#question-answering) for more information.
- This question answering pipeline can currently be loaded from [`pipeline`] using the following
- task identifier: `"question-answering"`.
+ This question answering pipeline can currently be loaded from [`pipeline`] using the following task identifier:
+ `"question-answering"`.
The models that this pipeline can use are models that have been fine-tuned on a question answering task. See the
- up-to-date list of available models on [huggingface.co/models](https://huggingface.co/models?filter=question-answering).
+ up-to-date list of available models on
+ [huggingface.co/models](https://huggingface.co/models?filter=question-answering).
"""
default_input_names = "question,context"
@@ -143,8 +145,8 @@ class QuestionAnsweringPipeline(ChunkPipeline):
question: Union[str, List[str]], context: Union[str, List[str]]
) -> Union[SquadExample, List[SquadExample]]:
"""
- QuestionAnsweringPipeline leverages the [`SquadExample`] internally. This helper method
- encapsulate all the logic for converting question(s) and context(s) to [`SquadExample`].
+ QuestionAnsweringPipeline leverages the [`SquadExample`] internally. This helper method encapsulate all the
+ logic for converting question(s) and context(s) to [`SquadExample`].
We currently support extractive question answering.
@@ -153,8 +155,7 @@ class QuestionAnsweringPipeline(ChunkPipeline):
context (`str` or `List[str]`): The context(s) in which we will look for the answer.
Returns:
- One or a list of [`SquadExample`]: The corresponding [`SquadExample`]
- grouping question and context.
+ One or a list of [`SquadExample`]: The corresponding [`SquadExample`] grouping question and context.
"""
if isinstance(question, list):
return [SquadExample(None, q, c, None, None, None) for q, c in zip(question, context)]
@@ -207,11 +208,11 @@ class QuestionAnsweringPipeline(ChunkPipeline):
args ([`SquadExample`] or a list of [`SquadExample`]):
One or several [`SquadExample`] containing the question and context.
X ([`SquadExample`] or a list of [`SquadExample`], *optional*):
- One or several [`SquadExample`] containing the question and context (will be treated
- the same way as if passed as the first positional argument).
+ One or several [`SquadExample`] containing the question and context (will be treated the same way as if
+ passed as the first positional argument).
data ([`SquadExample`] or a list of [`SquadExample`], *optional*):
- One or several [`SquadExample`] containing the question and context (will be treated
- the same way as if passed as the first positional argument).
+ One or several [`SquadExample`] containing the question and context (will be treated the same way as if
+ passed as the first positional argument).
question (`str` or `List[str]`):
One or several question(s) (must be used in conjunction with the `context` argument).
context (`str` or `List[str]`):
@@ -237,8 +238,7 @@ class QuestionAnsweringPipeline(ChunkPipeline):
A `dict` or a list of `dict`: Each result comes as a dictionary with the following keys:
- **score** (`float`) -- The probability associated to the answer.
- - **start** (`int`) -- The character start index of the answer (in the tokenized version of the
- input).
+ - **start** (`int`) -- The character start index of the answer (in the tokenized version of the input).
- **end** (`int`) -- The character end index of the answer (in the tokenized version of the input).
- **answer** (`str`) -- The answer to the question.
"""
diff --git a/src/transformers/pipelines/table_question_answering.py b/src/transformers/pipelines/table_question_answering.py
index fcce75b827..e50c492702 100644
--- a/src/transformers/pipelines/table_question_answering.py
+++ b/src/transformers/pipelines/table_question_answering.py
@@ -85,11 +85,12 @@ class TableQuestionAnsweringPipeline(Pipeline):
Table Question Answering pipeline using a `ModelForTableQuestionAnswering`. This pipeline is only available in
PyTorch.
- This tabular question answering pipeline can currently be loaded from [`pipeline`] using the
- following task identifier: `"table-question-answering"`.
+ This tabular question answering pipeline can currently be loaded from [`pipeline`] using the following task
+ identifier: `"table-question-answering"`.
The models that this pipeline can use are models that have been fine-tuned on a tabular question answering task.
- See the up-to-date list of available models on [huggingface.co/models](https://huggingface.co/models?filter=table-question-answering).
+ See the up-to-date list of available models on
+ [huggingface.co/models](https://huggingface.co/models?filter=table-question-answering).
"""
default_input_names = "table,query"
@@ -287,29 +288,29 @@ class TableQuestionAnsweringPipeline(Pipeline):
padding (`bool`, `str` or [`~file_utils.PaddingStrategy`], *optional*, defaults to `False`):
Activates and controls padding. Accepts the following values:
- - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a
- single sequence if provided).
- - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the
- maximum acceptable input length for the model if that argument is not provided.
- - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of
- different lengths).
+ - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
+ sequence if provided).
+ - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
+ acceptable input length for the model if that argument is not provided.
+ - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different
+ lengths).
truncation (`bool`, `str` or [`TapasTruncationStrategy`], *optional*, defaults to `False`):
Activates and controls truncation. Accepts the following values:
- - `True` or `'drop_rows_to_fit'`: Truncate to a maximum length specified with the argument
- `max_length` or to the maximum acceptable input length for the model if that argument is not
- provided. This will truncate row by row, removing rows from the table.
- - `False` or `'do_not_truncate'` (default): No truncation (i.e., can output batch with
- sequence lengths greater than the model maximum admissible input size).
+ - `True` or `'drop_rows_to_fit'`: Truncate to a maximum length specified with the argument `max_length`
+ or to the maximum acceptable input length for the model if that argument is not provided. This will
+ truncate row by row, removing rows from the table.
+ - `False` or `'do_not_truncate'` (default): No truncation (i.e., can output batch with sequence lengths
+ greater than the model maximum admissible input size).
Return:
A dictionary or a list of dictionaries containing results: Each result is a dictionary with the following
keys:
- - **answer** (`str`) -- The answer of the query given the table. If there is an aggregator, the answer
- will be preceded by `AGGREGATOR >`.
+ - **answer** (`str`) -- The answer of the query given the table. If there is an aggregator, the answer will
+ be preceded by `AGGREGATOR >`.
- **coordinates** (`List[Tuple[int, int]]`) -- Coordinates of the cells of the answers.
- **cells** (`List[str]`) -- List of strings made up of the answer cell values.
- **aggregator** (`str`) -- If the model has an aggregator, this returns the aggregator.
diff --git a/src/transformers/pipelines/text2text_generation.py b/src/transformers/pipelines/text2text_generation.py
index 69bff60a80..e6670a2f7a 100644
--- a/src/transformers/pipelines/text2text_generation.py
+++ b/src/transformers/pipelines/text2text_generation.py
@@ -27,11 +27,12 @@ class Text2TextGenerationPipeline(Pipeline):
"""
Pipeline for text to text generation using seq2seq models.
- This Text2TextGenerationPipeline pipeline can currently be loaded from [`pipeline`] using the
- following task identifier: `"text2text-generation"`.
+ This Text2TextGenerationPipeline pipeline can currently be loaded from [`pipeline`] using the following task
+ identifier: `"text2text-generation"`.
The models that this pipeline can use are models that have been fine-tuned on a translation task. See the
- up-to-date list of available models on [huggingface.co/models](https://huggingface.co/models?filter=text2text-generation).
+ up-to-date list of available models on
+ [huggingface.co/models](https://huggingface.co/models?filter=text2text-generation).
Usage:
@@ -119,9 +120,9 @@ class Text2TextGenerationPipeline(Pipeline):
clean_up_tokenization_spaces (`bool`, *optional*, defaults to `False`):
Whether or not to clean up the potential extra spaces in the text output.
truncation (`TruncationStrategy`, *optional*, defaults to `TruncationStrategy.DO_NOT_TRUNCATE`):
- The truncation strategy for the tokenization within the pipeline.
- `TruncationStrategy.DO_NOT_TRUNCATE` (default) will never truncate, but it is sometimes desirable
- to truncate the input to fit the model's max_length instead of throwing an error down the line.
+ The truncation strategy for the tokenization within the pipeline. `TruncationStrategy.DO_NOT_TRUNCATE`
+ (default) will never truncate, but it is sometimes desirable to truncate the input to fit the model's
+ max_length instead of throwing an error down the line.
generate_kwargs:
Additional keyword arguments to pass along to the generate method of the model (see the generate method
corresponding to your framework [here](./model#generative-models)).
@@ -130,8 +131,8 @@ class Text2TextGenerationPipeline(Pipeline):
A list or a list of list of `dict`: Each result comes as a dictionary with the following keys:
- **generated_text** (`str`, present when `return_text=True`) -- The generated text.
- - **generated_token_ids** (`torch.Tensor` or `tf.Tensor`, present when `return_tensors=True`)
- -- The token ids of the generated text.
+ - **generated_token_ids** (`torch.Tensor` or `tf.Tensor`, present when `return_tensors=True`) -- The token
+ ids of the generated text.
"""
result = super().__call__(*args, **kwargs)
@@ -175,8 +176,8 @@ class SummarizationPipeline(Text2TextGenerationPipeline):
"""
Summarize news articles and other documents.
- This summarizing pipeline can currently be loaded from [`pipeline`] using the following task
- identifier: `"summarization"`.
+ This summarizing pipeline can currently be loaded from [`pipeline`] using the following task identifier:
+ `"summarization"`.
The models that this pipeline can use are models that have been fine-tuned on a summarization task, which is
currently, '*bart-large-cnn*', '*t5-small*', '*t5-base*', '*t5-large*', '*t5-3b*', '*t5-11b*'. See the up-to-date
@@ -217,10 +218,9 @@ class SummarizationPipeline(Text2TextGenerationPipeline):
Return:
A list or a list of list of `dict`: Each result comes as a dictionary with the following keys:
- - **summary_text** (`str`, present when `return_text=True`) -- The summary of the corresponding
- input.
- - **summary_token_ids** (`torch.Tensor` or `tf.Tensor`, present when `return_tensors=True`) --
- The token ids of the summary.
+ - **summary_text** (`str`, present when `return_text=True`) -- The summary of the corresponding input.
+ - **summary_token_ids** (`torch.Tensor` or `tf.Tensor`, present when `return_tensors=True`) -- The token
+ ids of the summary.
"""
return super().__call__(*args, **kwargs)
@@ -243,8 +243,8 @@ class TranslationPipeline(Text2TextGenerationPipeline):
"""
Translates from one language to another.
- This translation pipeline can currently be loaded from [`pipeline`] using the following task
- identifier: `"translation_xx_to_yy"`.
+ This translation pipeline can currently be loaded from [`pipeline`] using the following task identifier:
+ `"translation_xx_to_yy"`.
The models that this pipeline can use are models that have been fine-tuned on a translation task. See the
up-to-date list of available models on [huggingface.co/models](https://huggingface.co/models?filter=translation).
@@ -318,7 +318,7 @@ class TranslationPipeline(Text2TextGenerationPipeline):
A list or a list of list of `dict`: Each result comes as a dictionary with the following keys:
- **translation_text** (`str`, present when `return_text=True`) -- The translation.
- - **translation_token_ids** (`torch.Tensor` or `tf.Tensor`, present when `return_tensors=True`)
- -- The token ids of the translation.
+ - **translation_token_ids** (`torch.Tensor` or `tf.Tensor`, present when `return_tensors=True`) -- The
+ token ids of the translation.
"""
return super().__call__(*args, **kwargs)
diff --git a/src/transformers/pipelines/text_classification.py b/src/transformers/pipelines/text_classification.py
index e3af7265a7..c2730304c1 100644
--- a/src/transformers/pipelines/text_classification.py
+++ b/src/transformers/pipelines/text_classification.py
@@ -37,8 +37,8 @@ class ClassificationFunction(ExplicitEnum):
function_to_apply (`str`, *optional*, defaults to `"default"`):
The function to apply to the model outputs in order to retrieve the scores. Accepts four different values:
- - `"default"`: if the model has a single label, will apply the sigmoid function on the output. If the
- model has several labels, will apply the softmax function on the output.
+ - `"default"`: if the model has a single label, will apply the sigmoid function on the output. If the model
+ has several labels, will apply the softmax function on the output.
- `"sigmoid"`: Applies the sigmoid function on the output.
- `"softmax"`: Applies the softmax function on the output.
- `"none"`: Does not apply any function on the output.
@@ -49,15 +49,15 @@ class TextClassificationPipeline(Pipeline):
Text classification pipeline using any `ModelForSequenceClassification`. See the [sequence classification
examples](../task_summary#sequence-classification) for more information.
- This text classification pipeline can currently be loaded from [`pipeline`] using the following
- task identifier: `"sentiment-analysis"` (for classifying sequences according to positive or negative
- sentiments).
+ This text classification pipeline can currently be loaded from [`pipeline`] using the following task identifier:
+ `"sentiment-analysis"` (for classifying sequences according to positive or negative sentiments).
- If multiple classification labels are available (`model.config.num_labels >= 2`), the pipeline will run a
- softmax over the results. If there is a single label, the pipeline will run a sigmoid over the result.
+ If multiple classification labels are available (`model.config.num_labels >= 2`), the pipeline will run a softmax
+ over the results. If there is a single label, the pipeline will run a sigmoid over the result.
The models that this pipeline can use are models that have been fine-tuned on a sequence classification task. See
- the up-to-date list of available models on [huggingface.co/models](https://huggingface.co/models?filter=text-classification).
+ the up-to-date list of available models on
+ [huggingface.co/models](https://huggingface.co/models?filter=text-classification).
"""
return_all_scores = False
diff --git a/src/transformers/pipelines/text_generation.py b/src/transformers/pipelines/text_generation.py
index 9ae77c3486..885c1f8da7 100644
--- a/src/transformers/pipelines/text_generation.py
+++ b/src/transformers/pipelines/text_generation.py
@@ -18,8 +18,8 @@ class TextGenerationPipeline(Pipeline):
Language generation pipeline using any `ModelWithLMHead`. This pipeline predicts the words that will follow a
specified text prompt.
- This language generation pipeline can currently be loaded from [`pipeline`] using the following
- task identifier: `"text-generation"`.
+ This language generation pipeline can currently be loaded from [`pipeline`] using the following task identifier:
+ `"text-generation"`.
The models that this pipeline can use are models that have been trained with an autoregressive language modeling
objective, which includes the uni-directional models in the library (e.g. gpt2). See the list of available models
@@ -141,8 +141,8 @@ class TextGenerationPipeline(Pipeline):
return_text (`bool`, *optional*, defaults to `True`):
Whether or not to include the decoded texts in the outputs.
return_full_text (`bool`, *optional*, defaults to `True`):
- If set to `False` only added text is returned, otherwise the full text is returned Only meaningful
- if *return_text* is set to True.
+ If set to `False` only added text is returned, otherwise the full text is returned Only meaningful if
+ *return_text* is set to True.
clean_up_tokenization_spaces (`bool`, *optional*, defaults to `False`):
Whether or not to clean up the potential extra spaces in the text output.
prefix (`str`, *optional*):
@@ -165,8 +165,8 @@ class TextGenerationPipeline(Pipeline):
A list or a list of list of `dict`: Each result comes as a dictionary with the following keys:
- **generated_text** (`str`, present when `return_text=True`) -- The generated text.
- - **generated_token_ids** (`torch.Tensor` or `tf.Tensor`, present when `return_tensors=True`)
- -- The token ids of the generated text.
+ - **generated_token_ids** (`torch.Tensor` or `tf.Tensor`, present when `return_tensors=True`) -- The token
+ ids of the generated text.
"""
return super().__call__(text_inputs, **kwargs)
diff --git a/src/transformers/pipelines/token_classification.py b/src/transformers/pipelines/token_classification.py
index 13d7014693..d14616a9aa 100644
--- a/src/transformers/pipelines/token_classification.py
+++ b/src/transformers/pipelines/token_classification.py
@@ -59,8 +59,8 @@ class AggregationStrategy(ExplicitEnum):
ignore_labels (`List[str]`, defaults to `["O"]`):
A list of labels to ignore.
grouped_entities (`bool`, *optional*, defaults to `False`):
- DEPRECATED, use `aggregation_strategy` instead. Whether or not to group the tokens corresponding to
- the same entity together in the predictions or not.
+ DEPRECATED, use `aggregation_strategy` instead. Whether or not to group the tokens corresponding to the
+ same entity together in the predictions or not.
aggregation_strategy (`str`, *optional*, defaults to `"none"`):
The strategy to fuse (or not) tokens based on the model prediction.
@@ -73,14 +73,14 @@ class AggregationStrategy(ExplicitEnum):
"NAME"}]. Look for FIRST, MAX, AVERAGE for ways to mitigate that and disambiguate words (on languages
that support that meaning, which is basically tokens separated by a space). These mitigations will
only work on real words, "New york" might still be tagged with two different entities.
- - "first" : (works only on word based models) Will use the `SIMPLE` strategy except that words,
- cannot end up with different tags. Words will simply use the tag of the first token of the word when
- there is ambiguity.
+ - "first" : (works only on word based models) Will use the `SIMPLE` strategy except that words, cannot
+ end up with different tags. Words will simply use the tag of the first token of the word when there
+ is ambiguity.
- "average" : (works only on word based models) Will use the `SIMPLE` strategy except that words,
cannot end up with different tags. scores will be averaged first across tokens, and then the maximum
label is applied.
- - "max" : (works only on word based models) Will use the `SIMPLE` strategy except that words,
- cannot end up with different tags. Word entity will simply be the token with the maximum score.
+ - "max" : (works only on word based models) Will use the `SIMPLE` strategy except that words, cannot
+ end up with different tags. Word entity will simply be the token with the maximum score.
""",
)
class TokenClassificationPipeline(Pipeline):
@@ -88,12 +88,12 @@ class TokenClassificationPipeline(Pipeline):
Named Entity Recognition pipeline using any `ModelForTokenClassification`. See the [named entity recognition
examples](../task_summary#named-entity-recognition) for more information.
- This token recognition pipeline can currently be loaded from [`pipeline`] using the following
- task identifier: `"ner"` (for predicting the classes of tokens in a sequence: person, organisation, location
- or miscellaneous).
+ This token recognition pipeline can currently be loaded from [`pipeline`] using the following task identifier:
+ `"ner"` (for predicting the classes of tokens in a sequence: person, organisation, location or miscellaneous).
The models that this pipeline can use are models that have been fine-tuned on a token classification task. See the
- up-to-date list of available models on [huggingface.co/models](https://huggingface.co/models?filter=token-classification).
+ up-to-date list of available models on
+ [huggingface.co/models](https://huggingface.co/models?filter=token-classification).
"""
default_input_names = "sequences"
@@ -166,20 +166,20 @@ class TokenClassificationPipeline(Pipeline):
One or several texts (or one list of texts) for token classification.
Return:
- A list or a list of list of `dict`: Each result comes as a list of dictionaries (one for each token in
- the corresponding input, or each entity if this pipeline was instantiated with an aggregation_strategy)
- with the following keys:
+ A list or a list of list of `dict`: Each result comes as a list of dictionaries (one for each token in the
+ corresponding input, or each entity if this pipeline was instantiated with an aggregation_strategy) with
+ the following keys:
- **word** (`str`) -- The token/word classified.
- **score** (`float`) -- The corresponding probability for `entity`.
- **entity** (`str`) -- The entity predicted for that token/word (it is named *entity_group* when
*aggregation_strategy* is not `"none"`.
- - **index** (`int`, only present when `aggregation_strategy="none"`) -- The index of the
- corresponding token in the sentence.
- - **start** (`int`, *optional*) -- The index of the start of the corresponding entity in the sentence.
- Only exists if the offsets are available within the tokenizer
- - **end** (`int`, *optional*) -- The index of the end of the corresponding entity in the sentence.
- Only exists if the offsets are available within the tokenizer
+ - **index** (`int`, only present when `aggregation_strategy="none"`) -- The index of the corresponding
+ token in the sentence.
+ - **start** (`int`, *optional*) -- The index of the start of the corresponding entity in the sentence. Only
+ exists if the offsets are available within the tokenizer
+ - **end** (`int`, *optional*) -- The index of the end of the corresponding entity in the sentence. Only
+ exists if the offsets are available within the tokenizer
"""
_inputs, offset_mapping = self._args_parser(inputs, **kwargs)
diff --git a/src/transformers/pipelines/zero_shot_classification.py b/src/transformers/pipelines/zero_shot_classification.py
index 3903805d63..bcc4c22f02 100644
--- a/src/transformers/pipelines/zero_shot_classification.py
+++ b/src/transformers/pipelines/zero_shot_classification.py
@@ -143,8 +143,8 @@ class ZeroShotClassificationPipeline(ChunkPipeline):
**kwargs,
):
"""
- Classify the sequence(s) given as inputs. See the [`ZeroShotClassificationPipeline`]
- documentation for more information.
+ Classify the sequence(s) given as inputs. See the [`ZeroShotClassificationPipeline`] documentation for more
+ information.
Args:
sequences (`str` or `List[str]`):
@@ -155,13 +155,13 @@ class ZeroShotClassificationPipeline(ChunkPipeline):
hypothesis_template (`str`, *optional*, defaults to `"This example is {}."`):
The template used to turn each label into an NLI-style hypothesis. This template must include a {} or
similar syntax for the candidate label to be inserted into the template. For example, the default
- template is `"This example is {}."` With the candidate label `"sports"`, this would be fed
- into the model like `" sequence to classify This example is sports . "`. The
- default template works well in many cases, but it may be worthwhile to experiment with different
- templates depending on the task setting.
+ template is `"This example is {}."` With the candidate label `"sports"`, this would be fed into the
+ model like `" sequence to classify This example is sports . "`. The default template
+ works well in many cases, but it may be worthwhile to experiment with different templates depending on
+ the task setting.
multi_label (`bool`, *optional*, defaults to `False`):
- Whether or not multiple candidate labels can be true. If `False`, the scores are normalized such
- that the sum of the label likelihoods for each sequence is 1. If `True`, the labels are considered
+ Whether or not multiple candidate labels can be true. If `False`, the scores are normalized such that
+ the sum of the label likelihoods for each sequence is 1. If `True`, the labels are considered
independent and probabilities are normalized for each candidate by doing a softmax of the entailment
score vs. the contradiction score.
diff --git a/src/transformers/testing_utils.py b/src/transformers/testing_utils.py
index d56c516ad0..fb912012c4 100644
--- a/src/transformers/testing_utils.py
+++ b/src/transformers/testing_utils.py
@@ -696,8 +696,8 @@ class CaptureStd:
err (`bool`, *optional*, defaults to `True`): Whether to capture stderr or not.
replay (`bool`, *optional*, defaults to `True`): Whether to replay or not.
By default each captured stream gets replayed back on context's exit, so that one can see what the test was
- doing. If this is a not wanted behavior and the captured data shouldn't be replayed, pass `replay=False`
- to disable this feature.
+ doing. If this is a not wanted behavior and the captured data shouldn't be replayed, pass `replay=False` to
+ disable this feature.
Examples:
@@ -955,14 +955,14 @@ class TestCasePlus(unittest.TestCase):
`after=False`: the temporary dir will always be left intact at the end of the test.
Note 1: In order to run the equivalent of `rm -r` safely, only subdirs of the project repository checkout are
- allowed if an explicit `tmp_dir` is used, so that by mistake no `/tmp` or similar important part of the
- filesystem will get nuked. i.e. please always pass paths that start with `./`
+ allowed if an explicit `tmp_dir` is used, so that by mistake no `/tmp` or similar important part of the filesystem
+ will get nuked. i.e. please always pass paths that start with `./`
Note 2: Each test can register multiple temporary dirs and they all will get auto-removed, unless requested
otherwise.
- Feature 3: Get a copy of the `os.environ` object that sets up `PYTHONPATH` specific to the current test suite.
- This is useful for invoking external programs from the test suite - e.g. distributed training.
+ Feature 3: Get a copy of the `os.environ` object that sets up `PYTHONPATH` specific to the current test suite. This
+ is useful for invoking external programs from the test suite - e.g. distributed training.
```python
@@ -1040,12 +1040,11 @@ class TestCasePlus(unittest.TestCase):
def get_env(self):
"""
- Return a copy of the `os.environ` object that sets up `PYTHONPATH` correctly, depending on the test suite
- it's invoked from. This is useful for invoking external programs from the test suite - e.g. distributed
- training.
+ Return a copy of the `os.environ` object that sets up `PYTHONPATH` correctly, depending on the test suite it's
+ invoked from. This is useful for invoking external programs from the test suite - e.g. distributed training.
- It always inserts `./src` first, then `./tests` or `./examples` depending on the test suite type and
- finally the preset `PYTHONPATH` if any (all full resolved paths).
+ It always inserts `./src` first, then `./tests` or `./examples` depending on the test suite type and finally
+ the preset `PYTHONPATH` if any (all full resolved paths).
"""
env = os.environ.copy()
@@ -1074,15 +1073,14 @@ class TestCasePlus(unittest.TestCase):
- sets `before=True` if `before` is `None`
- sets `after=False` if `after` is `None`
before (`bool`, *optional*):
- If `True` and the `tmp_dir` already exists, make sure to empty it right away if `False`
- and the `tmp_dir` already exists, any existing files will remain there.
+ If `True` and the `tmp_dir` already exists, make sure to empty it right away if `False` and the
+ `tmp_dir` already exists, any existing files will remain there.
after (`bool`, *optional*):
- If `True`, delete the `tmp_dir` at the end of the test if `False`, leave the
- `tmp_dir` and its contents intact at the end of the test.
+ If `True`, delete the `tmp_dir` at the end of the test if `False`, leave the `tmp_dir` and its contents
+ intact at the end of the test.
Returns:
- tmp_dir(`string`): either the same value as passed via *tmp_dir* or the path to the auto-selected tmp
- dir
+ tmp_dir(`string`): either the same value as passed via *tmp_dir* or the path to the auto-selected tmp dir
"""
if tmp_dir is not None:
@@ -1141,10 +1139,8 @@ def mockenv(**kwargs):
"""
this is a convenience wrapper, that allows this ::
- @mockenv(RUN_SLOW=True, USE_TF=False)
- def test_something():
- run_slow = os.getenv("RUN_SLOW", False)
- use_tf = os.getenv("USE_TF", False)
+ @mockenv(RUN_SLOW=True, USE_TF=False) def test_something():
+ run_slow = os.getenv("RUN_SLOW", False) use_tf = os.getenv("USE_TF", False)
"""
return mock.patch.dict(os.environ, kwargs)
@@ -1223,9 +1219,9 @@ def pytest_terminal_summary_main(tr, id):
- id: unique id like `tests` or `examples` that will be incorporated into the final reports filenames - this is
needed as some jobs have multiple runs of pytest, so we can't have them overwrite each other.
- NB: this functions taps into a private _pytest API and while unlikely, it could break should
- pytest do internal changes - also it calls default internal methods of terminalreporter which
- can be hijacked by various `pytest-` plugins and interfere.
+ NB: this functions taps into a private _pytest API and while unlikely, it could break should pytest do internal
+ changes - also it calls default internal methods of terminalreporter which can be hijacked by various `pytest-`
+ plugins and interfere.
"""
from _pytest.config import create_terminal_writer
@@ -1425,8 +1421,8 @@ def execute_subprocess_async(cmd, env=None, stdin=None, timeout=180, quiet=False
def pytest_xdist_worker_id():
"""
- Returns an int value of worker's numerical id under `pytest-xdist`'s concurrent workers `pytest -n N` regime,
- or 0 if `-n 1` or `pytest-xdist` isn't being used.
+ Returns an int value of worker's numerical id under `pytest-xdist`'s concurrent workers `pytest -n N` regime, or 0
+ if `-n 1` or `pytest-xdist` isn't being used.
"""
worker = os.environ.get("PYTEST_XDIST_WORKER", "gw0")
worker = re.sub(r"^gw", "", worker, 0, re.M)
@@ -1437,8 +1433,8 @@ def get_torch_dist_unique_port():
"""
Returns a port number that can be fed to `torch.distributed.launch`'s `--master_port` argument.
- Under `pytest-xdist` it adds a delta number based on a worker id so that concurrent tests don't try to use the
- same port at once.
+ Under `pytest-xdist` it adds a delta number based on a worker id so that concurrent tests don't try to use the same
+ port at once.
"""
port = 29500
uniq_delta = pytest_xdist_worker_id()
diff --git a/src/transformers/tokenization_utils.py b/src/transformers/tokenization_utils.py
index fafb962d63..90d6a1d2f3 100644
--- a/src/transformers/tokenization_utils.py
+++ b/src/transformers/tokenization_utils.py
@@ -454,8 +454,8 @@ class PreTrainedTokenizer(PreTrainedTokenizerBase):
- This encodes a dummy input and checks the number of added tokens, and is therefore not efficient. Do not
- put this inside your training loop.
+ This encodes a dummy input and checks the number of added tokens, and is therefore not efficient. Do not put
+ this inside your training loop.
diff --git a/src/transformers/tokenization_utils_base.py b/src/transformers/tokenization_utils_base.py
index a006efdae4..8d9182a30c 100644
--- a/src/transformers/tokenization_utils_base.py
+++ b/src/transformers/tokenization_utils_base.py
@@ -124,8 +124,8 @@ _re_tokenizer_file = re.compile(r"tokenizer\.(.*)\.json")
class TruncationStrategy(ExplicitEnum):
"""
- Possible values for the `truncation` argument in [`PreTrainedTokenizerBase.__call__`]. Useful for
- tab-completion in an IDE.
+ Possible values for the `truncation` argument in [`PreTrainedTokenizerBase.__call__`]. Useful for tab-completion in
+ an IDE.
"""
ONLY_FIRST = "only_first"
@@ -163,8 +163,7 @@ class TokenSpan(NamedTuple):
class BatchEncoding(UserDict):
"""
Holds the output of the [`~tokenization_utils_base.PreTrainedTokenizerBase.encode_plus`] and
- [`~tokenization_utils_base.PreTrainedTokenizerBase.batch_encode`] methods (tokens,
- attention_masks, etc).
+ [`~tokenization_utils_base.PreTrainedTokenizerBase.batch_encode`] methods (tokens, attention_masks, etc).
This class is derived from a python dictionary and can be used as a dictionary. In addition, this class exposes
utility methods to map from word/character space to token space.
@@ -213,16 +212,16 @@ class BatchEncoding(UserDict):
def n_sequences(self) -> Optional[int]:
"""
`Optional[int]`: The number of sequences used to generate each sample from the batch encoded in this
- [`BatchEncoding`]. Currently can be one of `None` (unknown), `1` (a single
- sentence) or `2` (a pair of sentences)
+ [`BatchEncoding`]. Currently can be one of `None` (unknown), `1` (a single sentence) or `2` (a pair of
+ sentences)
"""
return self._n_sequences
@property
def is_fast(self) -> bool:
"""
- `bool`: Indicate whether this [`BatchEncoding`] was generated from the result of a
- [`PreTrainedTokenizerFast`] or not.
+ `bool`: Indicate whether this [`BatchEncoding`] was generated from the result of a [`PreTrainedTokenizerFast`]
+ or not.
"""
return self._encodings is not None
@@ -275,8 +274,8 @@ class BatchEncoding(UserDict):
@property
def encodings(self) -> Optional[List[EncodingFast]]:
"""
- `Optional[List[tokenizers.Encoding]]`: The list all encodings from the tokenization process. Returns
- `None` if the input was tokenized through Python (i.e., not a fast) tokenizer.
+ `Optional[List[tokenizers.Encoding]]`: The list all encodings from the tokenization process. Returns `None` if
+ the input was tokenized through Python (i.e., not a fast) tokenizer.
"""
return self._encodings
@@ -308,9 +307,9 @@ class BatchEncoding(UserDict):
batch_index (`int`, *optional*, defaults to 0): The index to access in the batch.
Returns:
- `List[Optional[int]]`: A list indicating the sequence id corresponding to each token. Special tokens
- added by the tokenizer are mapped to `None` and other tokens are mapped to the index of their
- corresponding sequence.
+ `List[Optional[int]]`: A list indicating the sequence id corresponding to each token. Special tokens added
+ by the tokenizer are mapped to `None` and other tokens are mapped to the index of their corresponding
+ sequence.
"""
if not self._encodings:
raise ValueError("sequence_ids() is not available when using Python-based tokenizers")
@@ -324,9 +323,9 @@ class BatchEncoding(UserDict):
batch_index (`int`, *optional*, defaults to 0): The index to access in the batch.
Returns:
- `List[Optional[int]]`: A list indicating the word corresponding to each token. Special tokens added by
- the tokenizer are mapped to `None` and other tokens are mapped to the index of their corresponding
- word (several tokens will be mapped to the same word index if they are parts of that word).
+ `List[Optional[int]]`: A list indicating the word corresponding to each token. Special tokens added by the
+ tokenizer are mapped to `None` and other tokens are mapped to the index of their corresponding word
+ (several tokens will be mapped to the same word index if they are parts of that word).
"""
if not self._encodings:
raise ValueError("words() is not available when using Python-based tokenizers")
@@ -345,9 +344,9 @@ class BatchEncoding(UserDict):
batch_index (`int`, *optional*, defaults to 0): The index to access in the batch.
Returns:
- `List[Optional[int]]`: A list indicating the word corresponding to each token. Special tokens added by
- the tokenizer are mapped to `None` and other tokens are mapped to the index of their corresponding
- word (several tokens will be mapped to the same word index if they are parts of that word).
+ `List[Optional[int]]`: A list indicating the word corresponding to each token. Special tokens added by the
+ tokenizer are mapped to `None` and other tokens are mapped to the index of their corresponding word
+ (several tokens will be mapped to the same word index if they are parts of that word).
"""
if not self._encodings:
raise ValueError("word_ids() is not available when using Python-based tokenizers")
@@ -355,8 +354,8 @@ class BatchEncoding(UserDict):
def token_to_sequence(self, batch_or_token_index: int, token_index: Optional[int] = None) -> int:
"""
- Get the index of the sequence represented by the given token. In the general use case, this method returns
- `0` for a single sequence or the first sequence of a pair, and `1` for the second sequence of a pair
+ Get the index of the sequence represented by the given token. In the general use case, this method returns `0`
+ for a single sequence or the first sequence of a pair, and `1` for the second sequence of a pair
Can be called as:
@@ -444,8 +443,8 @@ class BatchEncoding(UserDict):
Can be called as:
- `self.word_to_tokens(word_index, sequence_index: int = 0)` if batch size is 1
- - `self.word_to_tokens(batch_index, word_index, sequence_index: int = 0)` if batch size is greater or equal
- to 1
+ - `self.word_to_tokens(batch_index, word_index, sequence_index: int = 0)` if batch size is greater or equal to
+ 1
This method is particularly suited when the input sequences are provided as pre-tokenized sequences (i.e. words
are defined by the user). In this case it allows to easily associate encoded tokens with provided tokenized
@@ -463,8 +462,8 @@ class BatchEncoding(UserDict):
or 1) the provided word index belongs to.
Returns:
- Optional [`~tokenization_utils_base.TokenSpan`] Span of tokens in the encoded sequence.
- Returns `None` if no tokens correspond to the word.
+ Optional [`~tokenization_utils_base.TokenSpan`] Span of tokens in the encoded sequence. Returns `None` if
+ no tokens correspond to the word.
"""
if not self._encodings:
@@ -586,8 +585,8 @@ class BatchEncoding(UserDict):
or 1) the provided word index belongs to.
Returns:
- `CharSpan` or `List[CharSpan]`: Span(s) of the associated character or characters in the string.
- CharSpan are NamedTuple with:
+ `CharSpan` or `List[CharSpan]`: Span(s) of the associated character or characters in the string. CharSpan
+ are NamedTuple with:
- start: index of the first character associated to the token in the original string
- end: index of the character following the last character associated to the token in the original
@@ -749,10 +748,9 @@ class BatchEncoding(UserDict):
class SpecialTokensMixin:
"""
- A mixin derived by [`PreTrainedTokenizer`] and [`PreTrainedTokenizerFast`] to
- handle specific behaviors related to special tokens. In particular, this class hold the attributes which can be
- used to directly access these special tokens in a model-independent manner and allow to set and update the special
- tokens.
+ A mixin derived by [`PreTrainedTokenizer`] and [`PreTrainedTokenizerFast`] to handle specific behaviors related to
+ special tokens. In particular, this class hold the attributes which can be used to directly access these special
+ tokens in a model-independent manner and allow to set and update the special tokens.
Args:
bos_token (`str` or `tokenizers.AddedToken`, *optional*):
@@ -834,9 +832,8 @@ class SpecialTokensMixin:
special tokens are NOT in the vocabulary, they are added to it (indexed starting from the last index of the
current vocabulary).
- Note,None
- When adding new tokens to the vocabulary, you should make sure to also resize the token embedding matrix of
- the model so that its embedding matrix matches the tokenizer.
+ Note,None When adding new tokens to the vocabulary, you should make sure to also resize the token embedding
+ matrix of the model so that its embedding matrix matches the tokenizer.
In order to do that, please use the [`~PreTrainedModel.resize_token_embeddings`] method.
@@ -847,14 +844,13 @@ class SpecialTokensMixin:
makes it easy to develop model-agnostic training and fine-tuning scripts.
When possible, special tokens are already registered for provided pretrained models (for instance
- [`BertTokenizer`] `cls_token` is already registered to be :obj*'[CLS]'* and XLM's one
- is also registered to be `''`).
+ [`BertTokenizer`] `cls_token` is already registered to be :obj*'[CLS]'* and XLM's one is also registered to be
+ `''`).
Args:
special_tokens_dict (dictionary *str* to *str* or `tokenizers.AddedToken`):
- Keys should be in the list of predefined special attributes: [`bos_token`, `eos_token`,
- `unk_token`, `sep_token`, `pad_token`, `cls_token`, `mask_token`,
- `additional_special_tokens`].
+ Keys should be in the list of predefined special attributes: [`bos_token`, `eos_token`, `unk_token`,
+ `sep_token`, `pad_token`, `cls_token`, `mask_token`, `additional_special_tokens`].
Tokens are only added if they are not already in the vocabulary (tested by checking if the tokenizer
assign the index of the `unk_token` to them).
@@ -909,18 +905,17 @@ class SpecialTokensMixin:
Add a list of new tokens to the tokenizer class. If the new tokens are not in the vocabulary, they are added to
it with indices starting from length of the current vocabulary.
- Note,None
- When adding new tokens to the vocabulary, you should make sure to also resize the token embedding matrix of
- the model so that its embedding matrix matches the tokenizer.
+ Note,None When adding new tokens to the vocabulary, you should make sure to also resize the token embedding
+ matrix of the model so that its embedding matrix matches the tokenizer.
In order to do that, please use the [`~PreTrainedModel.resize_token_embeddings`] method.
Args:
new_tokens (`str`, `tokenizers.AddedToken` or a list of *str* or `tokenizers.AddedToken`):
- Tokens are only added if they are not already in the vocabulary. `tokenizers.AddedToken` wraps a
- string token to let you personalize its behavior: whether this token should only match against a single
- word, whether this token should strip all potential whitespaces on the left side, whether this token
- should strip all potential whitespaces on the right side, etc.
+ Tokens are only added if they are not already in the vocabulary. `tokenizers.AddedToken` wraps a string
+ token to let you personalize its behavior: whether this token should only match against a single word,
+ whether this token should strip all potential whitespaces on the left side, whether this token should
+ strip all potential whitespaces on the right side, etc.
special_tokens (`bool`, *optional*, defaults to `False`):
Can be used to specify if the token is a special token. This mostly change the normalization behavior
(special tokens like CLS or [MASK] are usually not lower-cased for instance).
@@ -986,8 +981,8 @@ class SpecialTokensMixin:
@property
def sep_token(self) -> str:
"""
- `str`: Separation token, to separate context and query in an input sequence. Log an error if used while
- not having been set.
+ `str`: Separation token, to separate context and query in an input sequence. Log an error if used while not
+ having been set.
"""
if self._sep_token is None and self.verbose:
logger.error("Using sep_token, but it is not set yet.")
@@ -1007,8 +1002,8 @@ class SpecialTokensMixin:
@property
def cls_token(self) -> str:
"""
- `str`: Classification token, to extract a summary of an input sequence leveraging self-attention along the
- full depth of the model. Log an error if used while not having been set.
+ `str`: Classification token, to extract a summary of an input sequence leveraging self-attention along the full
+ depth of the model. Log an error if used while not having been set.
"""
if self._cls_token is None and self.verbose:
logger.error("Using cls_token, but it is not set yet.")
@@ -1018,8 +1013,8 @@ class SpecialTokensMixin:
@property
def mask_token(self) -> str:
"""
- `str`: Mask token, to use when training a model with masked-language modeling. Log an error if used while
- not having been set.
+ `str`: Mask token, to use when training a model with masked-language modeling. Log an error if used while not
+ having been set.
"""
if self._mask_token is None and self.verbose:
logger.error("Using mask_token, but it is not set yet.")
@@ -1029,8 +1024,8 @@ class SpecialTokensMixin:
@property
def additional_special_tokens(self) -> List[str]:
"""
- `List[str]`: All the additional special tokens you may want to use. Log an error if used while not having
- been set.
+ `List[str]`: All the additional special tokens you may want to use. Log an error if used while not having been
+ set.
"""
if self._additional_special_tokens is None and self.verbose:
logger.error("Using additional_special_tokens, but it is not set yet.")
@@ -1072,8 +1067,8 @@ class SpecialTokensMixin:
@property
def bos_token_id(self) -> Optional[int]:
"""
- `Optional[int]`: Id of the beginning of sentence token in the vocabulary. Returns `None` if the token
- has not been set.
+ `Optional[int]`: Id of the beginning of sentence token in the vocabulary. Returns `None` if the token has not
+ been set.
"""
if self._bos_token is None:
return None
@@ -1082,8 +1077,8 @@ class SpecialTokensMixin:
@property
def eos_token_id(self) -> Optional[int]:
"""
- `Optional[int]`: Id of the end of sentence token in the vocabulary. Returns `None` if the token has
- not been set.
+ `Optional[int]`: Id of the end of sentence token in the vocabulary. Returns `None` if the token has not been
+ set.
"""
if self._eos_token is None:
return None
@@ -1092,8 +1087,7 @@ class SpecialTokensMixin:
@property
def unk_token_id(self) -> Optional[int]:
"""
- `Optional[int]`: Id of the unknown token in the vocabulary. Returns `None` if the token has not been
- set.
+ `Optional[int]`: Id of the unknown token in the vocabulary. Returns `None` if the token has not been set.
"""
if self._unk_token is None:
return None
@@ -1112,8 +1106,7 @@ class SpecialTokensMixin:
@property
def pad_token_id(self) -> Optional[int]:
"""
- `Optional[int]`: Id of the padding token in the vocabulary. Returns `None` if the token has not been
- set.
+ `Optional[int]`: Id of the padding token in the vocabulary. Returns `None` if the token has not been set.
"""
if self._pad_token is None:
return None
@@ -1129,8 +1122,8 @@ class SpecialTokensMixin:
@property
def cls_token_id(self) -> Optional[int]:
"""
- `Optional[int]`: Id of the classification token in the vocabulary, to extract a summary of an input
- sequence leveraging self-attention along the full depth of the model.
+ `Optional[int]`: Id of the classification token in the vocabulary, to extract a summary of an input sequence
+ leveraging self-attention along the full depth of the model.
Returns `None` if the token has not been set.
"""
@@ -1151,8 +1144,8 @@ class SpecialTokensMixin:
@property
def additional_special_tokens_ids(self) -> List[int]:
"""
- `List[int]`: Ids of all the additional special tokens in the vocabulary. Log an error if used while not
- having been set.
+ `List[int]`: Ids of all the additional special tokens in the vocabulary. Log an error if used while not having
+ been set.
"""
return self.convert_tokens_to_ids(self.additional_special_tokens)
@@ -1210,12 +1203,11 @@ class SpecialTokensMixin:
@property
def special_tokens_map_extended(self) -> Dict[str, Union[str, AddedToken, List[Union[str, AddedToken]]]]:
"""
- `Dict[str, Union[str, tokenizers.AddedToken, List[Union[str, tokenizers.AddedToken]]]]`: A dictionary
- mapping special token class attributes (`cls_token`, `unk_token`, etc.) to their values
- (`''`, `''`, etc.).
+ `Dict[str, Union[str, tokenizers.AddedToken, List[Union[str, tokenizers.AddedToken]]]]`: A dictionary mapping
+ special token class attributes (`cls_token`, `unk_token`, etc.) to their values (`''`, `''`, etc.).
- Don't convert tokens of `tokenizers.AddedToken` type to string so they can be used to control more finely
- how special tokens are tokenized.
+ Don't convert tokens of `tokenizers.AddedToken` type to string so they can be used to control more finely how
+ special tokens are tokenized.
"""
set_attr = {}
for attr in self.SPECIAL_TOKENS_ATTRIBUTES:
@@ -1237,11 +1229,11 @@ class SpecialTokensMixin:
@property
def all_special_tokens_extended(self) -> List[Union[str, AddedToken]]:
"""
- `List[Union[str, tokenizers.AddedToken]]`: All the special tokens (`''`, `''`, etc.)
- mapped to class attributes.
+ `List[Union[str, tokenizers.AddedToken]]`: All the special tokens (`''`, `''`, etc.) mapped to class
+ attributes.
- Don't convert tokens of `tokenizers.AddedToken` type to string so they can be used to control more finely
- how special tokens are tokenized.
+ Don't convert tokens of `tokenizers.AddedToken` type to string so they can be used to control more finely how
+ special tokens are tokenized.
"""
all_toks = []
set_attr = self.special_tokens_map_extended
@@ -1253,8 +1245,7 @@ class SpecialTokensMixin:
@property
def all_special_ids(self) -> List[int]:
"""
- `List[int]`: List the ids of the special tokens(`''`, `''`, etc.) mapped to class
- attributes.
+ `List[int]`: List the ids of the special tokens(`''`, `''`, etc.) mapped to class attributes.
"""
all_toks = self.all_special_tokens
all_ids = self.convert_tokens_to_ids(all_toks)
@@ -1267,33 +1258,33 @@ ENCODE_KWARGS_DOCSTRING = r"""
padding (`bool`, `str` or [`~file_utils.PaddingStrategy`], *optional*, defaults to `False`):
Activates and controls padding. Accepts the following values:
- - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a
- single sequence if provided).
- - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the
- maximum acceptable input length for the model if that argument is not provided.
- - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of
- different lengths).
+ - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
+ sequence if provided).
+ - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
+ acceptable input length for the model if that argument is not provided.
+ - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different
+ lengths).
truncation (`bool`, `str` or [`~tokenization_utils_base.TruncationStrategy`], *optional*, defaults to `False`):
Activates and controls truncation. Accepts the following values:
- - `True` or `'longest_first'`: Truncate to a maximum length specified with the argument
- `max_length` or to the maximum acceptable input length for the model if that argument is not
- provided. This will truncate token by token, removing a token from the longest sequence in the pair
- if a pair of sequences (or a batch of pairs) is provided.
- - `'only_first'`: Truncate to a maximum length specified with the argument `max_length` or to
- the maximum acceptable input length for the model if that argument is not provided. This will only
+ - `True` or `'longest_first'`: Truncate to a maximum length specified with the argument `max_length` or
+ to the maximum acceptable input length for the model if that argument is not provided. This will
+ truncate token by token, removing a token from the longest sequence in the pair if a pair of
+ sequences (or a batch of pairs) is provided.
+ - `'only_first'`: Truncate to a maximum length specified with the argument `max_length` or to the
+ maximum acceptable input length for the model if that argument is not provided. This will only
truncate the first sequence of a pair if a pair of sequences (or a batch of pairs) is provided.
- - `'only_second'`: Truncate to a maximum length specified with the argument `max_length` or
- to the maximum acceptable input length for the model if that argument is not provided. This will only
+ - `'only_second'`: Truncate to a maximum length specified with the argument `max_length` or to the
+ maximum acceptable input length for the model if that argument is not provided. This will only
truncate the second sequence of a pair if a pair of sequences (or a batch of pairs) is provided.
- - `False` or `'do_not_truncate'` (default): No truncation (i.e., can output batch with
- sequence lengths greater than the model maximum admissible input size).
+ - `False` or `'do_not_truncate'` (default): No truncation (i.e., can output batch with sequence lengths
+ greater than the model maximum admissible input size).
max_length (`int`, *optional*):
Controls the maximum length to use by one of the truncation/padding parameters.
- If left unset or set to `None`, this will use the predefined model maximum length if a maximum
- length is required by one of the truncation/padding parameters. If the model has no specific maximum
- input length (like XLNet) truncation/padding to a maximum length will be deactivated.
+ If left unset or set to `None`, this will use the predefined model maximum length if a maximum length
+ is required by one of the truncation/padding parameters. If the model has no specific maximum input
+ length (like XLNet) truncation/padding to a maximum length will be deactivated.
stride (`int`, *optional*, defaults to 0):
If set to a number along with `max_length`, the overflowing tokens returned when
`return_overflowing_tokens=True` will contain some tokens from the end of the truncated sequence
@@ -1327,16 +1318,15 @@ ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING = r"""
[What are attention masks?](../glossary#attention-mask)
return_overflowing_tokens (`bool`, *optional*, defaults to `False`):
Whether or not to return overflowing token sequences. If a pair of sequences of input ids (or a batch
- of pairs) is provided with `truncation_strategy = longest_first` or `True`, an error is
- raised instead of returning overflowing tokens.
+ of pairs) is provided with `truncation_strategy = longest_first` or `True`, an error is raised instead
+ of returning overflowing tokens.
return_special_tokens_mask (`bool`, *optional*, defaults to `False`):
Whether or not to return special tokens mask information.
return_offsets_mapping (`bool`, *optional*, defaults to `False`):
Whether or not to return `(char_start, char_end)` for each token.
- This is only available on fast tokenizers inheriting from
- [`PreTrainedTokenizerFast`], if using Python's tokenizer, this method will raise
- `NotImplementedError`.
+ This is only available on fast tokenizers inheriting from [`PreTrainedTokenizerFast`], if using
+ Python's tokenizer, this method will raise `NotImplementedError`.
return_length (`bool`, *optional*, defaults to `False`):
Whether or not to return the lengths of the encoded inputs.
verbose (`bool`, *optional*, defaults to `True`):
@@ -1350,8 +1340,8 @@ ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING = r"""
[What are input IDs?](../glossary#input-ids)
- - **token_type_ids** -- List of token type ids to be fed to a model (when `return_token_type_ids=True`
- or if *"token_type_ids"* is in `self.model_input_names`).
+ - **token_type_ids** -- List of token type ids to be fed to a model (when `return_token_type_ids=True` or
+ if *"token_type_ids"* is in `self.model_input_names`).
[What are token type IDs?](../glossary#token-type-ids)
@@ -1372,31 +1362,30 @@ ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING = r"""
INIT_TOKENIZER_DOCSTRING = r"""
Class attributes (overridden by derived classes)
- - **vocab_files_names** (`Dict[str, str]`) -- A dictionary with, as keys, the `__init__` keyword name of
- each vocabulary file required by the model, and as associated values, the filename for saving the associated
- file (string).
+ - **vocab_files_names** (`Dict[str, str]`) -- A dictionary with, as keys, the `__init__` keyword name of each
+ vocabulary file required by the model, and as associated values, the filename for saving the associated file
+ (string).
- **pretrained_vocab_files_map** (`Dict[str, Dict[str, str]]`) -- A dictionary of dictionaries, with the
high-level keys being the `__init__` keyword name of each vocabulary file required by the model, the
- low-level being the `short-cut-names` of the pretrained models with, as associated values, the
- `url` to the associated pretrained vocabulary file.
- - **max_model_input_sizes** (`Dict[str, Optional[int]]`) -- A dictionary with, as keys, the
- `short-cut-names` of the pretrained models, and as associated values, the maximum length of the sequence
- inputs of this model, or `None` if the model has no maximum input size.
+ low-level being the `short-cut-names` of the pretrained models with, as associated values, the `url` to the
+ associated pretrained vocabulary file.
+ - **max_model_input_sizes** (`Dict[str, Optional[int]]`) -- A dictionary with, as keys, the `short-cut-names`
+ of the pretrained models, and as associated values, the maximum length of the sequence inputs of this model,
+ or `None` if the model has no maximum input size.
- **pretrained_init_configuration** (`Dict[str, Dict[str, Any]]`) -- A dictionary with, as keys, the
- `short-cut-names` of the pretrained models, and as associated values, a dictionary of specific arguments
- to pass to the `__init__` method of the tokenizer class for this pretrained model when loading the
- tokenizer with the [`~tokenization_utils_base.PreTrainedTokenizerBase.from_pretrained`]
- method.
+ `short-cut-names` of the pretrained models, and as associated values, a dictionary of specific arguments to
+ pass to the `__init__` method of the tokenizer class for this pretrained model when loading the tokenizer
+ with the [`~tokenization_utils_base.PreTrainedTokenizerBase.from_pretrained`] method.
- **model_input_names** (`List[str]`) -- A list of inputs expected in the forward pass of the model.
- - **padding_side** (`str`) -- The default value for the side on which the model should have padding
- applied. Should be `'right'` or `'left'`.
+ - **padding_side** (`str`) -- The default value for the side on which the model should have padding applied.
+ Should be `'right'` or `'left'`.
Args:
model_max_length (`int`, *optional*):
The maximum length (in number of tokens) for the inputs to the transformer model. When the tokenizer is
- loaded with [`~tokenization_utils_base.PreTrainedTokenizerBase.from_pretrained`], this
- will be set to the value stored for the associated model in `max_model_input_sizes` (see above). If no
- value is provided, will default to VERY_LARGE_INTEGER (`int(1e30)`).
+ loaded with [`~tokenization_utils_base.PreTrainedTokenizerBase.from_pretrained`], this will be set to the
+ value stored for the associated model in `max_model_input_sizes` (see above). If no value is provided, will
+ default to VERY_LARGE_INTEGER (`int(1e30)`).
padding_side: (`str`, *optional*):
The side on which the model should have padding applied. Should be selected between ['right', 'left'].
Default value is picked from the class attribute of the same name.
@@ -1417,8 +1406,7 @@ INIT_TOKENIZER_DOCSTRING = r"""
associated to `self.sep_token` and `self.sep_token_id`.
pad_token (`str` or `tokenizers.AddedToken`, *optional*):
A special token used to make arrays of tokens the same size for batching purpose. Will then be ignored by
- attention mechanisms or loss computation. Will be associated to `self.pad_token` and
- `self.pad_token_id`.
+ attention mechanisms or loss computation. Will be associated to `self.pad_token` and `self.pad_token_id`.
cls_token (`str` or `tokenizers.AddedToken`, *optional*):
A special token representing the class of the input (used by BERT for instance). Will be associated to
`self.cls_token` and `self.cls_token_id`.
@@ -1528,8 +1516,8 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin):
"""
Returns the vocabulary as a dictionary of token to index.
- `tokenizer.get_vocab()[token]` is equivalent to `tokenizer.convert_tokens_to_ids(token)` when
- `token` is in the vocab.
+ `tokenizer.get_vocab()[token]` is equivalent to `tokenizer.convert_tokens_to_ids(token)` when `token` is in the
+ vocab.
Returns:
`Dict[str, int]`: The vocabulary.
@@ -1539,8 +1527,8 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin):
@classmethod
def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], *init_inputs, **kwargs):
r"""
- Instantiate a [`~tokenization_utils_base.PreTrainedTokenizerBase`] (or a derived class) from
- a predefined tokenizer.
+ Instantiate a [`~tokenization_utils_base.PreTrainedTokenizerBase`] (or a derived class) from a predefined
+ tokenizer.
Args:
pretrained_model_name_or_path (`str` or `os.PathLike`):
@@ -1550,8 +1538,8 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin):
Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under a
user or organization name, like `dbmdz/bert-base-german-cased`.
- A path to a *directory* containing vocabulary files required by the tokenizer, for instance saved
- using the [`~tokenization_utils_base.PreTrainedTokenizerBase.save_pretrained`]
- method, e.g., `./my_model_directory/`.
+ using the [`~tokenization_utils_base.PreTrainedTokenizerBase.save_pretrained`] method, e.g.,
+ `./my_model_directory/`.
- (**Deprecated**, not applicable to all derived classes) A path or url to a single saved vocabulary
file (if and only if the tokenizer only requires a single vocabulary file like Bert or XLNet), e.g.,
`./my_model_directory/vocab.txt`.
@@ -1565,10 +1553,11 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin):
Whether or not to delete incompletely received files. Attempt to resume the download if such a file
exists.
proxies (`Dict[str, str]`, *optional*):
- A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request.
+ A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',
+ 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request.
use_auth_token (`str` or *bool*, *optional*):
- The token to use as HTTP bearer authorization for remote files. If `True`, will use the token
- generated when running `transformers-cli login` (stored in `~/.huggingface`).
+ The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated
+ when running `transformers-cli login` (stored in `~/.huggingface`).
local_files_only (`bool`, *optional*, defaults to `False`):
Whether or not to only rely on local files and not to attempt to download any files.
revision(`str`, *optional*, defaults to `"main"`):
@@ -1581,9 +1570,9 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin):
inputs (additional positional arguments, *optional*):
Will be passed along to the Tokenizer `__init__` method.
kwargs (additional keyword arguments, *optional*):
- Will be passed to the Tokenizer `__init__` method. Can be used to set special tokens like
- `bos_token`, `eos_token`, `unk_token`, `sep_token`, `pad_token`, `cls_token`,
- `mask_token`, `additional_special_tokens`. See parameters in the `__init__` for more details.
+ Will be passed to the Tokenizer `__init__` method. Can be used to set special tokens like `bos_token`,
+ `eos_token`, `unk_token`, `sep_token`, `pad_token`, `cls_token`, `mask_token`,
+ `additional_special_tokens`. See parameters in the `__init__` for more details.
@@ -1961,9 +1950,8 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin):
This method make sure the full tokenizer can then be re-loaded using the
[`~tokenization_utils_base.PreTrainedTokenizer.from_pretrained`] class method..
- Warning,None
- This won't save modifications you may have applied to the tokenizer after the instantiation (for instance,
- modifying `tokenizer.do_lower_case` after creation).
+ Warning,None This won't save modifications you may have applied to the tokenizer after the instantiation (for
+ instance, modifying `tokenizer.do_lower_case` after creation).
Args:
save_directory (`str` or `os.PathLike`): The path to a directory where the tokenizer will be saved.
@@ -1972,12 +1960,12 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin):
format as well as in legacy format if it exists, i.e. with tokenizer specific vocabulary and a separate
added_tokens files.
- If `False`, will only save the tokenizer in the unified JSON format. This format is incompatible
- with "slow" tokenizers (not powered by the *tokenizers* library), so the tokenizer will not be able to
- be loaded in the corresponding "slow" tokenizer.
+ If `False`, will only save the tokenizer in the unified JSON format. This format is incompatible with
+ "slow" tokenizers (not powered by the *tokenizers* library), so the tokenizer will not be able to be
+ loaded in the corresponding "slow" tokenizer.
- If `True`, will save the tokenizer in legacy format. If the "slow" tokenizer doesn't exits, a
- value error is raised.
+ If `True`, will save the tokenizer in legacy format. If the "slow" tokenizer doesn't exits, a value
+ error is raised.
filename_prefix: (`str`, *optional*):
A prefix to add to the names of the files saved by the tokenizer.
push_to_hub (`bool`, *optional*, defaults to `False`):
@@ -1985,10 +1973,9 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin):
- Using `push_to_hub=True` will synchronize the repository you are pushing to with
- `save_directory`, which requires `save_directory` to be a local clone of the repo you are
- pushing to if it's an existing folder. Pass along `temp_dir=True` to use a temporary directory
- instead.
+ Using `push_to_hub=True` will synchronize the repository you are pushing to with `save_directory`,
+ which requires `save_directory` to be a local clone of the repo you are pushing to if it's an existing
+ folder. Pass along `temp_dir=True` to use a temporary directory instead.
@@ -2147,8 +2134,7 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin):
""",
"""
Returns:
- `List[int]`, `torch.Tensor`, `tf.Tensor` or `np.ndarray`: The tokenized ids of the
- text.
+ `List[int]`, `torch.Tensor`, `tf.Tensor` or `np.ndarray`: The tokenized ids of the text.
""",
)
def encode(
@@ -2175,8 +2161,8 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin):
method).
text_pair (`str`, `List[str]` or `List[int]`, *optional*):
Optional second sequence to be encoded. This can be a string, a list of strings (tokenized string using
- the `tokenize` method) or a list of integers (tokenized string ids using the
- `convert_tokens_to_ids` method).
+ the `tokenize` method) or a list of integers (tokenized string ids using the `convert_tokens_to_ids`
+ method).
"""
encoded_inputs = self.encode_plus(
text,
@@ -2494,8 +2480,8 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin):
method).
text_pair (`str`, `List[str]` or `List[int]`, *optional*):
Optional second sequence to be encoded. This can be a string, a list of strings (tokenized string using
- the `tokenize` method) or a list of integers (tokenized string ids using the
- `convert_tokens_to_ids` method).
+ the `tokenize` method) or a list of integers (tokenized string ids using the `convert_tokens_to_ids`
+ method).
"""
# Backward compatibility for 'truncation_strategy', 'pad_to_max_length'
@@ -2681,29 +2667,30 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin):
If the `encoded_inputs` passed are dictionary of numpy arrays, PyTorch tensors or TensorFlow tensors, the
- result will use the same type unless you provide a different tensor type with `return_tensors`. In the
- case of PyTorch tensors, you will lose the specific device of your tensors however.
+ result will use the same type unless you provide a different tensor type with `return_tensors`. In the case of
+ PyTorch tensors, you will lose the specific device of your tensors however.
Args:
encoded_inputs ([`BatchEncoding`], list of [`BatchEncoding`], `Dict[str, List[int]]`, `Dict[str, List[List[int]]` or `List[Dict[str, List[int]]]`):
- Tokenized inputs. Can represent one input ([`BatchEncoding`] or `Dict[str, List[int]]`) or a batch of tokenized inputs (list of [`BatchEncoding`], *Dict[str,
- List[List[int]]]* or *List[Dict[str, List[int]]]*) so you can use this method during preprocessing as
- well as in a PyTorch Dataloader collate function.
+ Tokenized inputs. Can represent one input ([`BatchEncoding`] or `Dict[str, List[int]]`) or a batch of
+ tokenized inputs (list of [`BatchEncoding`], *Dict[str, List[List[int]]]* or *List[Dict[str,
+ List[int]]]*) so you can use this method during preprocessing as well as in a PyTorch Dataloader
+ collate function.
- Instead of `List[int]` you can have tensors (numpy arrays, PyTorch tensors or TensorFlow tensors),
- see the note above for the return type.
+ Instead of `List[int]` you can have tensors (numpy arrays, PyTorch tensors or TensorFlow tensors), see
+ the note above for the return type.
padding (`bool`, `str` or [`~file_utils.PaddingStrategy`], *optional*, defaults to `True`):
Select a strategy to pad the returned sequences (according to the model's padding side and padding
index) among:
- - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a
- single sequence if provided).
- - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the
- maximum acceptable input length for the model if that argument is not provided.
- - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of
- different lengths).
+ - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
+ sequence if provided).
+ - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
+ acceptable input length for the model if that argument is not provided.
+ - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different
+ lengths).
max_length (`int`, *optional*):
Maximum length of the returned list and optionally padding length (see above).
pad_to_multiple_of (`int`, *optional*):
@@ -2819,7 +2806,8 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin):
self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
) -> List[int]:
"""
- Create the token type IDs corresponding to the sequences passed. [What are token type IDs?](../glossary#token-type-ids)
+ Create the token type IDs corresponding to the sequences passed. [What are token type
+ IDs?](../glossary#token-type-ids)
Should be overridden in a subclass if the model has a special way of building those.
@@ -2885,8 +2873,8 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin):
Args:
ids (`List[int]`):
- Tokenized input ids of the first sequence. Can be obtained from a string by chaining the `tokenize`
- and `convert_tokens_to_ids` methods.
+ Tokenized input ids of the first sequence. Can be obtained from a string by chaining the `tokenize` and
+ `convert_tokens_to_ids` methods.
pair_ids (`List[int]`, *optional*):
Tokenized input ids of the second sequence. Can be obtained from a string by chaining the `tokenize`
and `convert_tokens_to_ids` methods.
@@ -3003,8 +2991,8 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin):
Args:
ids (`List[int]`):
- Tokenized input ids of the first sequence. Can be obtained from a string by chaining the `tokenize`
- and `convert_tokens_to_ids` methods.
+ Tokenized input ids of the first sequence. Can be obtained from a string by chaining the `tokenize` and
+ `convert_tokens_to_ids` methods.
pair_ids (`List[int]`, *optional*):
Tokenized input ids of the second sequence. Can be obtained from a string by chaining the `tokenize`
and `convert_tokens_to_ids` methods.
@@ -3013,26 +3001,26 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin):
truncation_strategy (`str` or [`~tokenization_utils_base.TruncationStrategy`], *optional*, defaults to `False`):
The strategy to follow for truncation. Can be:
- - `'longest_first'`: Truncate to a maximum length specified with the argument `max_length` or
- to the maximum acceptable input length for the model if that argument is not provided. This will
- truncate token by token, removing a token from the longest sequence in the pair if a pair of
- sequences (or a batch of pairs) is provided.
- - `'only_first'`: Truncate to a maximum length specified with the argument `max_length` or to
- the maximum acceptable input length for the model if that argument is not provided. This will only
+ - `'longest_first'`: Truncate to a maximum length specified with the argument `max_length` or to the
+ maximum acceptable input length for the model if that argument is not provided. This will truncate
+ token by token, removing a token from the longest sequence in the pair if a pair of sequences (or a
+ batch of pairs) is provided.
+ - `'only_first'`: Truncate to a maximum length specified with the argument `max_length` or to the
+ maximum acceptable input length for the model if that argument is not provided. This will only
truncate the first sequence of a pair if a pair of sequences (or a batch of pairs) is provided.
- - `'only_second'`: Truncate to a maximum length specified with the argument `max_length` or
- to the maximum acceptable input length for the model if that argument is not provided. This will only
+ - `'only_second'`: Truncate to a maximum length specified with the argument `max_length` or to the
+ maximum acceptable input length for the model if that argument is not provided. This will only
truncate the second sequence of a pair if a pair of sequences (or a batch of pairs) is provided.
- - `'do_not_truncate'` (default): No truncation (i.e., can output batch with sequence lengths
- greater than the model maximum admissible input size).
+ - `'do_not_truncate'` (default): No truncation (i.e., can output batch with sequence lengths greater
+ than the model maximum admissible input size).
stride (`int`, *optional*, defaults to 0):
If set to a positive number, the overflowing tokens returned will contain some tokens from the main
sequence returned. The value of this argument defines the number of additional tokens.
Returns:
- `Tuple[List[int], List[int], List[int]]`: The truncated `ids`, the truncated `pair_ids` and the
- list of overflowing tokens. Note: The *longest_first* strategy returns empty list of overflowing tokens if
- a pair of sequences (or a batch of pairs) is provided.
+ `Tuple[List[int], List[int], List[int]]`: The truncated `ids`, the truncated `pair_ids` and the list of
+ overflowing tokens. Note: The *longest_first* strategy returns empty list of overflowing tokens if a pair
+ of sequences (or a batch of pairs) is provided.
"""
if num_tokens_to_remove <= 0:
return ids, pair_ids, []
@@ -3098,7 +3086,8 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin):
Pad encoded inputs (on left/right and up to predefined length or max length in the batch)
Args:
- encoded_inputs: Dictionary of tokenized inputs (`List[int]`) or batch of tokenized inputs (`List[List[int]]`).
+ encoded_inputs:
+ Dictionary of tokenized inputs (`List[int]`) or batch of tokenized inputs (`List[List[int]]`).
max_length: maximum length of the returned list and optionally padding length (see below).
Will truncate by taking into account the special tokens.
padding_strategy: PaddingStrategy to use for padding.
@@ -3113,7 +3102,8 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin):
pad_to_multiple_of: (optional) Integer if set will pad the sequence to a multiple of the provided value.
This is especially useful to enable the use of Tensor Core on NVIDIA hardware with compute capability
>= 7.5 (Volta).
- return_attention_mask: (optional) Set to False to avoid returning attention mask (default: set to model specifics)
+ return_attention_mask:
+ (optional) Set to False to avoid returning attention mask (default: set to model specifics)
"""
# Load from model defaults
if return_attention_mask is None:
@@ -3358,21 +3348,21 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin):
List of summaries or target language texts.
max_length (`int`, *optional*):
Controls the maximum length for encoder inputs (documents to summarize or source language texts) If
- left unset or set to `None`, this will use the predefined model maximum length if a maximum length
- is required by one of the truncation/padding parameters. If the model has no specific maximum input
- length (like XLNet) truncation/padding to a maximum length will be deactivated.
+ left unset or set to `None`, this will use the predefined model maximum length if a maximum length is
+ required by one of the truncation/padding parameters. If the model has no specific maximum input length
+ (like XLNet) truncation/padding to a maximum length will be deactivated.
max_target_length (`int`, *optional*):
Controls the maximum length of decoder inputs (target language texts or summaries) If left unset or set
to `None`, this will use the max_length value.
padding (`bool`, `str` or [`~file_utils.PaddingStrategy`], *optional*, defaults to `False`):
Activates and controls padding. Accepts the following values:
- - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a
- single sequence if provided).
- - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the
- maximum acceptable input length for the model if that argument is not provided.
- - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of
- different lengths).
+ - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
+ sequence if provided).
+ - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
+ acceptable input length for the model if that argument is not provided.
+ - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different
+ lengths).
return_tensors (`str` or [`~file_utils.TensorType`], *optional*):
If set, will return tensors instead of list of python integers. Acceptable values are:
@@ -3382,18 +3372,18 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin):
truncation (`bool`, `str` or [`~tokenization_utils_base.TruncationStrategy`], *optional*, defaults to `True`):
Activates and controls truncation. Accepts the following values:
- - `True` or `'longest_first'`: Truncate to a maximum length specified with the argument
- `max_length` or to the maximum acceptable input length for the model if that argument is not
- provided. This will truncate token by token, removing a token from the longest sequence in the pair
- if a pair of sequences (or a batch of pairs) is provided.
- - `'only_first'`: Truncate to a maximum length specified with the argument `max_length` or to
- the maximum acceptable input length for the model if that argument is not provided. This will only
+ - `True` or `'longest_first'`: Truncate to a maximum length specified with the argument `max_length` or
+ to the maximum acceptable input length for the model if that argument is not provided. This will
+ truncate token by token, removing a token from the longest sequence in the pair if a pair of
+ sequences (or a batch of pairs) is provided.
+ - `'only_first'`: Truncate to a maximum length specified with the argument `max_length` or to the
+ maximum acceptable input length for the model if that argument is not provided. This will only
truncate the first sequence of a pair if a pair of sequences (or a batch of pairs) is provided.
- - `'only_second'`: Truncate to a maximum length specified with the argument `max_length` or
- to the maximum acceptable input length for the model if that argument is not provided. This will only
+ - `'only_second'`: Truncate to a maximum length specified with the argument `max_length` or to the
+ maximum acceptable input length for the model if that argument is not provided. This will only
truncate the second sequence of a pair if a pair of sequences (or a batch of pairs) is provided.
- - `False` or `'do_not_truncate'` (default): No truncation (i.e., can output batch with
- sequence lengths greater than the model maximum admissible input size).
+ - `False` or `'do_not_truncate'` (default): No truncation (i.e., can output batch with sequence lengths
+ greater than the model maximum admissible input size).
**kwargs:
Additional keyword arguments passed along to `self.__call__`.
@@ -3474,8 +3464,8 @@ def get_fast_tokenizer_file(
git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any
identifier allowed by git.
use_auth_token (`str` or *bool*, *optional*):
- The token to use as HTTP bearer authorization for remote files. If `True`, will use the token
- generated when running `transformers-cli login` (stored in `~/.huggingface`).
+ The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated
+ when running `transformers-cli login` (stored in `~/.huggingface`).
local_files_only (`bool`, *optional*, defaults to `False`):
Whether or not to only rely on local files and not to attempt to download any files.
diff --git a/src/transformers/tokenization_utils_fast.py b/src/transformers/tokenization_utils_fast.py
index 88628efb58..7d39c9e818 100644
--- a/src/transformers/tokenization_utils_fast.py
+++ b/src/transformers/tokenization_utils_fast.py
@@ -57,11 +57,11 @@ ADDED_TOKENS_FILE = "added_tokens.json"
INIT_TOKENIZER_DOCSTRING += """
tokenizer_object ([`tokenizers.Tokenizer`]):
- A [`tokenizers.Tokenizer`] object from 🤗 tokenizers to instantiate from. See [Using tokenizers
- from 🤗 tokenizers](../fast_tokenizers) for more information.
+ A [`tokenizers.Tokenizer`] object from 🤗 tokenizers to instantiate from. See [Using tokenizers from 🤗
+ tokenizers](../fast_tokenizers) for more information.
tokenizer_file ([`str`]):
- A path to a local JSON file representing a previously serialized [`tokenizers.Tokenizer`] object from
- 🤗 tokenizers.
+ A path to a local JSON file representing a previously serialized [`tokenizers.Tokenizer`] object from 🤗
+ tokenizers.
"""
MODEL_TO_TRAINER_MAPPING = {
@@ -272,8 +272,8 @@ class PreTrainedTokenizerFast(PreTrainedTokenizerBase):
- This encodes a dummy input and checks the number of added tokens, and is therefore not efficient. Do not
- put this inside your training loop.
+ This encodes a dummy input and checks the number of added tokens, and is therefore not efficient. Do not put
+ this inside your training loop.
@@ -606,8 +606,8 @@ class PreTrainedTokenizerFast(PreTrainedTokenizerBase):
Additional keyword arguments passed along to the trainer from the 🤗 Tokenizers library.
Returns:
- [`PreTrainedTokenizerFast`]: A new tokenizer of the same type as the original one,
- trained on `text_iterator`.
+ [`PreTrainedTokenizerFast`]: A new tokenizer of the same type as the original one, trained on
+ `text_iterator`.
"""
tokenizer_json = json.loads(self._tokenizer.to_str())
diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py
index 0423c38197..59118c75c0 100755
--- a/src/transformers/trainer.py
+++ b/src/transformers/trainer.py
@@ -204,29 +204,28 @@ class Trainer:
- [`Trainer`] is optimized to work with the [`PreTrainedModel`]
- provided by the library. You can still use your own models defined as `torch.nn.Module` as long as
- they work the same way as the 🤗 Transformers models.
+ [`Trainer`] is optimized to work with the [`PreTrainedModel`] provided by the library. You can still use
+ your own models defined as `torch.nn.Module` as long as they work the same way as the 🤗 Transformers
+ models.
args ([`TrainingArguments`], *optional*):
- The arguments to tweak for training. Will default to a basic instance of
- [`TrainingArguments`] with the `output_dir` set to a directory named *tmp_trainer* in
- the current directory if not provided.
+ The arguments to tweak for training. Will default to a basic instance of [`TrainingArguments`] with the
+ `output_dir` set to a directory named *tmp_trainer* in the current directory if not provided.
data_collator (`DataCollator`, *optional*):
- The function to use to form a batch from a list of elements of `train_dataset` or `eval_dataset`.
- Will default to [`default_data_collator`] if no `tokenizer` is provided, an instance of
+ The function to use to form a batch from a list of elements of `train_dataset` or `eval_dataset`. Will
+ default to [`default_data_collator`] if no `tokenizer` is provided, an instance of
[`DataCollatorWithPadding`] otherwise.
train_dataset (`torch.utils.data.Dataset` or `torch.utils.data.IterableDataset`, *optional*):
The dataset to use for training. If it is an `datasets.Dataset`, columns not accepted by the
`model.forward()` method are automatically removed.
- Note that if it's a `torch.utils.data.IterableDataset` with some randomization and you are training in
- a distributed fashion, your iterable dataset should either use a internal attribute `generator` that
- is a `torch.Generator` for the randomization that must be identical on all processes (and the Trainer
- will manually set the seed of this `generator` at each epoch) or have a `set_epoch()` method that
- internally sets the seed of the RNGs used.
+ Note that if it's a `torch.utils.data.IterableDataset` with some randomization and you are training in a
+ distributed fashion, your iterable dataset should either use a internal attribute `generator` that is a
+ `torch.Generator` for the randomization that must be identical on all processes (and the Trainer will
+ manually set the seed of this `generator` at each epoch) or have a `set_epoch()` method that internally
+ sets the seed of the RNGs used.
eval_dataset (`torch.utils.data.Dataset`, *optional*):
The dataset to use for evaluation. If it is an `datasets.Dataset`, columns not accepted by the
`model.forward()` method are automatically removed.
@@ -235,40 +234,39 @@ class Trainer:
maximum length when batching inputs, and it will be saved along the model to make it easier to rerun an
interrupted training or reuse the fine-tuned model.
model_init (`Callable[[], PreTrainedModel]`, *optional*):
- A function that instantiates the model to be used. If provided, each call to
- [`~Trainer.train`] will start from a new instance of the model as given by this function.
+ A function that instantiates the model to be used. If provided, each call to [`~Trainer.train`] will start
+ from a new instance of the model as given by this function.
The function may have zero argument, or a single one containing the optuna/Ray Tune/SigOpt trial object, to
be able to choose different architectures according to hyper parameters (such as layer count, sizes of
inner layers, dropout probabilities etc).
compute_metrics (`Callable[[EvalPrediction], Dict]`, *optional*):
- The function that will be used to compute metrics at evaluation. Must take a
- [`EvalPrediction`] and return a dictionary string to metric values.
+ The function that will be used to compute metrics at evaluation. Must take a [`EvalPrediction`] and return
+ a dictionary string to metric values.
callbacks (List of [`TrainerCallback`], *optional*):
A list of callbacks to customize the training loop. Will add those to the list of default callbacks
detailed in [here](callback).
If you want to remove one of the default callbacks used, use the [`Trainer.remove_callback`] method.
optimizers (`Tuple[torch.optim.Optimizer, torch.optim.lr_scheduler.LambdaLR]`, *optional*): A tuple
- containing the optimizer and the scheduler to use. Will default to an instance of
- [`AdamW`] on your model and a scheduler given by
- [`get_linear_schedule_with_warmup`] controlled by `args`.
+ containing the optimizer and the scheduler to use. Will default to an instance of [`AdamW`] on your model
+ and a scheduler given by [`get_linear_schedule_with_warmup`] controlled by `args`.
Important attributes:
- - **model** -- Always points to the core model. If using a transformers model, it will be a
- [`PreTrainedModel`] subclass.
+ - **model** -- Always points to the core model. If using a transformers model, it will be a [`PreTrainedModel`]
+ subclass.
- **model_wrapped** -- Always points to the most external model in case one or more other modules wrap the
original model. This is the model that should be used for the forward pass. For example, under `DeepSpeed`,
- the inner model is wrapped in `DeepSpeed` and then again in `torch.nn.DistributedDataParallel`. If the
- inner model hasn't been wrapped, then `self.model_wrapped` is the same as `self.model`.
+ the inner model is wrapped in `DeepSpeed` and then again in `torch.nn.DistributedDataParallel`. If the inner
+ model hasn't been wrapped, then `self.model_wrapped` is the same as `self.model`.
- **is_model_parallel** -- Whether or not a model has been switched to a model parallel mode (different from
data parallelism, this means some of the model layers are split on different GPUs).
- **place_model_on_device** -- Whether or not to automatically place the model on the device - it will be set
to `False` if model parallel or deepspeed is used, or if the default
`TrainingArguments.place_model_on_device` is overridden to return `False` .
- - **is_in_train** -- Whether or not a model is currently running `train` (e.g. when `evaluate` is called
- while in `train`)
+ - **is_in_train** -- Whether or not a model is currently running `train` (e.g. when `evaluate` is called while
+ in `train`)
"""
@@ -497,8 +495,8 @@ class Trainer:
Args:
callback (`type` or [`~transformer.TrainerCallback`]):
- A [`~transformer.TrainerCallback`] class or an instance of a [`~transformer.TrainerCallback`].
- In the first case, will instantiate a member of that class.
+ A [`~transformer.TrainerCallback`] class or an instance of a [`~transformer.TrainerCallback`]. In the
+ first case, will instantiate a member of that class.
"""
self.callback_handler.add_callback(callback)
@@ -510,8 +508,8 @@ class Trainer:
Args:
callback (`type` or [`~transformer.TrainerCallback`]):
- A [`~transformer.TrainerCallback`] class or an instance of a [`~transformer.TrainerCallback`].
- In the first case, will pop the first member of that class found in the list of callbacks.
+ A [`~transformer.TrainerCallback`] class or an instance of a [`~transformer.TrainerCallback`]. In the
+ first case, will pop the first member of that class found in the list of callbacks.
Returns:
[`~transformer.TrainerCallback`]: The callback removed, if found.
@@ -524,8 +522,8 @@ class Trainer:
Args:
callback (`type` or [`~transformer.TrainerCallback`]):
- A [`~transformer.TrainerCallback`] class or an instance of a [`~transformer.TrainerCallback`].
- In the first case, will remove the first member of that class found in the list of callbacks.
+ A [`~transformer.TrainerCallback`] class or an instance of a [`~transformer.TrainerCallback`]. In the
+ first case, will remove the first member of that class found in the list of callbacks.
"""
self.callback_handler.remove_callback(callback)
@@ -629,8 +627,8 @@ class Trainer:
"""
Returns the training [`~torch.utils.data.DataLoader`].
- Will use no sampler if `self.train_dataset` does not implement `__len__`, a random sampler (adapted
- to distributed training if necessary) otherwise.
+ Will use no sampler if `self.train_dataset` does not implement `__len__`, a random sampler (adapted to
+ distributed training if necessary) otherwise.
Subclass and override this method if you want to inject some custom behavior.
"""
@@ -708,8 +706,8 @@ class Trainer:
Args:
eval_dataset (`torch.utils.data.Dataset`, *optional*):
- If provided, will override `self.eval_dataset`. If it is an `datasets.Dataset`, columns not
- accepted by the `model.forward()` method are automatically removed. It must implement `__len__`.
+ If provided, will override `self.eval_dataset`. If it is an `datasets.Dataset`, columns not accepted by
+ the `model.forward()` method are automatically removed. It must implement `__len__`.
"""
if eval_dataset is None and self.eval_dataset is None:
raise ValueError("Trainer: evaluation requires an eval_dataset.")
@@ -755,8 +753,8 @@ class Trainer:
Args:
test_dataset (`torch.utils.data.Dataset`, *optional*):
- The test dataset to use. If it is an `datasets.Dataset`, columns not accepted by the
- `model.forward()` method are automatically removed. It must implement `__len__`.
+ The test dataset to use. If it is an `datasets.Dataset`, columns not accepted by the `model.forward()`
+ method are automatically removed. It must implement `__len__`.
"""
if is_datasets_available() and isinstance(test_dataset, datasets.Dataset):
test_dataset = self._remove_unused_columns(test_dataset, description="test")
@@ -795,8 +793,8 @@ class Trainer:
Setup the optimizer and the learning rate scheduler.
We provide a reasonable default that works well. If you want to use something else, you can pass a tuple in the
- Trainer's init through `optimizers`, or subclass and override this method (or `create_optimizer`
- and/or `create_scheduler`) in a subclass.
+ Trainer's init through `optimizers`, or subclass and override this method (or `create_optimizer` and/or
+ `create_scheduler`) in a subclass.
"""
self.create_optimizer()
self.create_scheduler(num_training_steps=num_training_steps, optimizer=self.optimizer)
@@ -1035,10 +1033,9 @@ class Trainer:
Args:
resume_from_checkpoint (`str` or `bool`, *optional*):
- If a `str`, local path to a saved checkpoint as saved by a previous instance of
- [`Trainer`]. If a `bool` and equals *True*, load the last checkpoint in
- *args.output_dir* as saved by a previous instance of [`Trainer`]. If present,
- training will resume from the model/optimizer/scheduler states loaded here.
+ If a `str`, local path to a saved checkpoint as saved by a previous instance of [`Trainer`]. If a
+ `bool` and equals *True*, load the last checkpoint in *args.output_dir* as saved by a previous instance
+ of [`Trainer`]. If present, training will resume from the model/optimizer/scheduler states loaded here.
trial (`optuna.Trial` or `Dict[str, Any]`, *optional*):
The trial run or the hyperparameter dictionary for hyperparameter search.
ignore_keys_for_eval (`List[str]`, *optional*)
@@ -1720,42 +1717,41 @@ class Trainer:
**kwargs,
) -> BestRun:
"""
- Launch an hyperparameter search using `optuna` or `Ray Tune` or `SigOpt`. The optimized quantity is
- determined by `compute_objective`, which defaults to a function returning the evaluation loss when no
- metric is provided, the sum of all metrics otherwise.
+ Launch an hyperparameter search using `optuna` or `Ray Tune` or `SigOpt`. The optimized quantity is determined
+ by `compute_objective`, which defaults to a function returning the evaluation loss when no metric is provided,
+ the sum of all metrics otherwise.
- To use this method, you need to have provided a `model_init` when initializing your
- [`Trainer`]: we need to reinitialize the model at each new run. This is incompatible
- with the `optimizers` argument, so you need to subclass [`Trainer`] and override the
- method [`~Trainer.create_optimizer_and_scheduler`] for custom optimizer/scheduler.
+ To use this method, you need to have provided a `model_init` when initializing your [`Trainer`]: we need to
+ reinitialize the model at each new run. This is incompatible with the `optimizers` argument, so you need to
+ subclass [`Trainer`] and override the method [`~Trainer.create_optimizer_and_scheduler`] for custom
+ optimizer/scheduler.
Args:
hp_space (`Callable[["optuna.Trial"], Dict[str, float]]`, *optional*):
A function that defines the hyperparameter search space. Will default to
- [`~trainer_utils.default_hp_space_optuna`] or
- [`~trainer_utils.default_hp_space_ray`] or
+ [`~trainer_utils.default_hp_space_optuna`] or [`~trainer_utils.default_hp_space_ray`] or
[`~trainer_utils.default_hp_space_sigopt`] depending on your backend.
compute_objective (`Callable[[Dict[str, float]], float]`, *optional*):
- A function computing the objective to minimize or maximize from the metrics returned by the
- `evaluate` method. Will default to [`~trainer_utils.default_compute_objective`].
+ A function computing the objective to minimize or maximize from the metrics returned by the `evaluate`
+ method. Will default to [`~trainer_utils.default_compute_objective`].
n_trials (`int`, *optional*, defaults to 100):
The number of trial runs to test.
direction(`str`, *optional*, defaults to `"minimize"`):
- Whether to optimize greater or lower objects. Can be `"minimize"` or `"maximize"`, you should
- pick `"minimize"` when optimizing the validation loss, `"maximize"` when optimizing one or
- several metrics.
+ Whether to optimize greater or lower objects. Can be `"minimize"` or `"maximize"`, you should pick
+ `"minimize"` when optimizing the validation loss, `"maximize"` when optimizing one or several metrics.
backend(`str` or [`~training_utils.HPSearchBackend`], *optional*):
The backend to use for hyperparameter search. Will default to optuna or Ray Tune or SigOpt, depending
on which one is installed. If all are installed, will default to optuna.
kwargs:
- Additional keyword arguments passed along to `optuna.create_study` or `ray.tune.run`. For
- more information see:
+ Additional keyword arguments passed along to `optuna.create_study` or `ray.tune.run`. For more
+ information see:
- - the documentation of [optuna.create_study](https://optuna.readthedocs.io/en/stable/reference/generated/optuna.study.create_study.html)
+ - the documentation of
+ [optuna.create_study](https://optuna.readthedocs.io/en/stable/reference/generated/optuna.study.create_study.html)
- the documentation of [tune.run](https://docs.ray.io/en/latest/tune/api_docs/execution.html#tune-run)
- the documentation of [sigopt](https://app.sigopt.com/docs/endpoints/experiments/create)
@@ -2131,9 +2127,9 @@ class Trainer:
Args:
eval_dataset (`Dataset`, *optional*):
- Pass a dataset if you wish to override `self.eval_dataset`. If it is an `datasets.Dataset`,
- columns not accepted by the `model.forward()` method are automatically removed. It must implement the
- `__len__` method.
+ Pass a dataset if you wish to override `self.eval_dataset`. If it is an `datasets.Dataset`, columns not
+ accepted by the `model.forward()` method are automatically removed. It must implement the `__len__`
+ method.
ignore_keys (`Lst[str]`, *optional*):
A list of keys in the output of your model (if it is a dictionary) that should be ignored when
gathering predictions.
@@ -2206,9 +2202,9 @@ class Trainer:
- If your predictions or labels have different sequence length (for instance because you're doing dynamic
- padding in a token classification task) the predictions will be padded (on the right) to allow for
- concatenation into one array. The padding index is -100.
+ If your predictions or labels have different sequence length (for instance because you're doing dynamic padding
+ in a token classification task) the predictions will be padded (on the right) to allow for concatenation into
+ one array. The padding index is -100.
@@ -2216,8 +2212,8 @@ class Trainer:
- predictions (`np.ndarray`): The predictions on `test_dataset`.
- label_ids (`np.ndarray`, *optional*): The labels (if the dataset contained some).
- - metrics (`Dict[str, float]`, *optional*): The potential dictionary of metrics (if the dataset
- contained labels).
+ - metrics (`Dict[str, float]`, *optional*): The potential dictionary of metrics (if the dataset contained
+ labels).
"""
# memory metrics - must set up as early as possible
self._memory_tracker.start()
@@ -2561,9 +2557,9 @@ class Trainer:
def floating_point_ops(self, inputs: Dict[str, Union[torch.Tensor, Any]]):
"""
- For models that inherit from [`PreTrainedModel`], uses that method to compute the number of
- floating point operations for every backward + forward pass. If using another model, either implement such a
- method in the model or subclass and override this method.
+ For models that inherit from [`PreTrainedModel`], uses that method to compute the number of floating point
+ operations for every backward + forward pass. If using another model, either implement such a method in the
+ model or subclass and override this method.
Args:
inputs (`Dict[str, Union[torch.Tensor, Any]]`):
@@ -2707,8 +2703,8 @@ class Trainer:
Additional keyword arguments passed along to [`~Trainer.create_model_card`].
Returns:
- The url of the commit of your model in the given repository if `blocking=False`, a tuple with the url
- of the commit and an object to track the progress of the commit if `blocking=True`
+ The url of the commit of your model in the given repository if `blocking=False`, a tuple with the url of
+ the commit and an object to track the progress of the commit if `blocking=True`
"""
if self.args.should_save:
diff --git a/src/transformers/trainer_callback.py b/src/transformers/trainer_callback.py
index 1a4a0eae75..e2afa7e939 100644
--- a/src/transformers/trainer_callback.py
+++ b/src/transformers/trainer_callback.py
@@ -35,14 +35,14 @@ logger = logging.get_logger(__name__)
@dataclass
class TrainerState:
"""
- A class containing the [`Trainer`] inner state that will be saved along the model and optimizer
- when checkpointing and passed to the [`TrainerCallback`].
+ A class containing the [`Trainer`] inner state that will be saved along the model and optimizer when checkpointing
+ and passed to the [`TrainerCallback`].
- In all this class, one step is to be understood as one update step. When using gradient accumulation, one
- update step may require several forward and backward passes: if you use `gradient_accumulation_steps=n`,
- then one update step requires going through *n* batches.
+ In all this class, one step is to be understood as one update step. When using gradient accumulation, one update
+ step may require several forward and backward passes: if you use `gradient_accumulation_steps=n`, then one update
+ step requires going through *n* batches.
@@ -110,8 +110,8 @@ class TrainerState:
@dataclass
class TrainerControl:
"""
- A class that handles the [`Trainer`] control flow. This class is used by the
- [`TrainerCallback`] to activate some switches in the training loop.
+ A class that handles the [`Trainer`] control flow. This class is used by the [`TrainerCallback`] to activate some
+ switches in the training loop.
Args:
should_training_stop (`bool`, *optional*, defaults to `False`):
@@ -190,12 +190,12 @@ class TrainerCallback:
Those are only accessible in the event `on_log`.
- The `control` object is the only one that can be changed by the callback, in which case the event that changes
- it should return the modified version.
+ The `control` object is the only one that can be changed by the callback, in which case the event that changes it
+ should return the modified version.
- The argument `args`, `state` and `control` are positionals for all events, all the others are
- grouped in `kwargs`. You can unpack the ones you need in the signature of the event using them. As an example,
- see the code of the simple [`~transformer.PrinterCallback`].
+ The argument `args`, `state` and `control` are positionals for all events, all the others are grouped in `kwargs`.
+ You can unpack the ones you need in the signature of the event using them. As an example, see the code of the
+ simple [`~transformer.PrinterCallback`].
Example:
@@ -407,8 +407,7 @@ class CallbackHandler(TrainerCallback):
class DefaultFlowCallback(TrainerCallback):
"""
- A [`TrainerCallback`] that handles the default flow of the training loop for logs, evaluation
- and checkpoints.
+ A [`TrainerCallback`] that handles the default flow of the training loop for logs, evaluation and checkpoints.
"""
def on_step_end(self, args: TrainingArguments, state: TrainerState, control: TrainerControl, **kwargs):
@@ -514,11 +513,11 @@ class EarlyStoppingCallback(TrainerCallback):
Use with `metric_for_best_model` to stop training when the specified metric worsens for
`early_stopping_patience` evaluation calls.
early_stopping_threshold(`float`, *optional*):
- Use with TrainingArguments `metric_for_best_model` and `early_stopping_patience` to denote how
- much the specified metric must improve to satisfy early stopping conditions. `
+ Use with TrainingArguments `metric_for_best_model` and `early_stopping_patience` to denote how much the
+ specified metric must improve to satisfy early stopping conditions. `
- This callback depends on [`TrainingArguments`] argument *load_best_model_at_end* functionality
- to set best_metric in [`TrainerState`].
+ This callback depends on [`TrainingArguments`] argument *load_best_model_at_end* functionality to set best_metric
+ in [`TrainerState`].
"""
def __init__(self, early_stopping_patience: int = 1, early_stopping_threshold: Optional[float] = 0.0):
diff --git a/src/transformers/trainer_pt_utils.py b/src/transformers/trainer_pt_utils.py
index 0743fc610f..ddcc10e9c1 100644
--- a/src/transformers/trainer_pt_utils.py
+++ b/src/transformers/trainer_pt_utils.py
@@ -398,8 +398,8 @@ class DistributedTensorGatherer:
def add_arrays(self, arrays):
"""
- Add `arrays` to the internal storage, Will initialize the storage to the full size at the first arrays
- passed so that if we're bound to get an OOM, it happens at the beginning.
+ Add `arrays` to the internal storage, Will initialize the storage to the full size at the first arrays passed
+ so that if we're bound to get an OOM, it happens at the beginning.
"""
if arrays is None:
return
@@ -485,8 +485,8 @@ class LabelSmoother:
def get_length_grouped_indices(lengths, batch_size, mega_batch_mult=None, generator=None):
"""
- Return a list of indices so that each slice of `batch_size` consecutive indices correspond to elements of
- similar lengths. To do this, the indices are:
+ Return a list of indices so that each slice of `batch_size` consecutive indices correspond to elements of similar
+ lengths. To do this, the indices are:
- randomly permuted
- grouped in mega-batches of size `mega_batch_mult * batch_size`
@@ -641,11 +641,10 @@ class DistributedLengthGroupedSampler(DistributedSampler):
class ShardSampler(Sampler):
"""
Sampler that shards batches between several processes. Dispatches indices batch by batch: on 2 processes with batch
- size 4, the first two batches are `[0, 1, 2, 3, 4, 5, 6, 7]` and `[8, 9, 10, 11, 12, 13, 14, 15]`, which
- shard into `[0, 1, 2, 3]` and `[8, 9, 10, 11]` for GPU-0 and `[4, 5, 6, 7]` and `[12, 13, 14, 15]` for GPU-1.
+ size 4, the first two batches are `[0, 1, 2, 3, 4, 5, 6, 7]` and `[8, 9, 10, 11, 12, 13, 14, 15]`, which shard into
+ `[0, 1, 2, 3]` and `[8, 9, 10, 11]` for GPU-0 and `[4, 5, 6, 7]` and `[12, 13, 14, 15]` for GPU-1.
- The sampler thus yields `[0, 1, 2, 3, 8, 9, 10, 11]` on GPU-0 and `[4, 5, 6, 7, 12, 13, 14, 15]` on
- GPU-1.
+ The sampler thus yields `[0, 1, 2, 3, 8, 9, 10, 11]` on GPU-0 and `[4, 5, 6, 7, 12, 13, 14, 15]` on GPU-1.
"""
def __init__(
@@ -688,26 +687,25 @@ class ShardSampler(Sampler):
class IterableDatasetShard(IterableDataset):
"""
- Wraps a PyTorch `IterableDataset` to generate samples for one of the processes only. Instances of this class
- will always yield a number of samples that is a round multiple of the actual batch size (which is `batch_size x num_processes`). Depending on the value of the `drop_last` attribute, it will either stop the iteration at
- the first batch that would be too small or loop with indices from the beginning.
+ Wraps a PyTorch `IterableDataset` to generate samples for one of the processes only. Instances of this class will
+ always yield a number of samples that is a round multiple of the actual batch size (which is `batch_size x
+ num_processes`). Depending on the value of the `drop_last` attribute, it will either stop the iteration at the
+ first batch that would be too small or loop with indices from the beginning.
- On two processes with an iterable dataset yielding of `[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]` with a batch
- size of 2:
+ On two processes with an iterable dataset yielding of `[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]` with a batch size of
+ 2:
- - the shard on process 0 will yield `[0, 1, 4, 5, 8, 9]` so will see batches `[0, 1]`, `[4, 5]`,
- `[8, 9]`
- - the shard on process 1 will yield `[2, 3, 6, 7, 10, 11]` so will see batches `[2, 3]`, `[6, 7]`,
- `[10, 11]`
+ - the shard on process 0 will yield `[0, 1, 4, 5, 8, 9]` so will see batches `[0, 1]`, `[4, 5]`, `[8, 9]`
+ - the shard on process 1 will yield `[2, 3, 6, 7, 10, 11]` so will see batches `[2, 3]`, `[6, 7]`, `[10, 11]`
If your IterableDataset implements some randomization that needs to be applied the same way on all processes
- (for instance, a shuffling), you should use a `torch.Generator` in a `generator` attribute of the
- `dataset` to generate your random numbers and call the
- [`~trainer_pt_utils.IterableDatasetShard.set_epoch`] method of this object. It will set the
- seed of this `generator` to `seed + epoch` on all processes before starting the iteration.
- Alternatively, you can also implement a `set_epoch()` method in your iterable dataset to deal with this.
+ (for instance, a shuffling), you should use a `torch.Generator` in a `generator` attribute of the `dataset` to
+ generate your random numbers and call the [`~trainer_pt_utils.IterableDatasetShard.set_epoch`] method of this
+ object. It will set the seed of this `generator` to `seed + epoch` on all processes before starting the
+ iteration. Alternatively, you can also implement a `set_epoch()` method in your iterable dataset to deal with
+ this.
@@ -914,23 +912,22 @@ def log_metrics(self, split, metrics):
The GPU allocated and peak memory reporting is done with `torch.cuda.memory_allocated()` and
`torch.cuda.max_memory_allocated()`. This metric reports only "deltas" for pytorch-specific allocations, as
- `torch.cuda` memory management system doesn't track any memory allocated outside of pytorch. For example, the
- very first cuda call typically loads CUDA kernels, which may take from 0.5 to 2GB of GPU memory.
+ `torch.cuda` memory management system doesn't track any memory allocated outside of pytorch. For example, the very
+ first cuda call typically loads CUDA kernels, which may take from 0.5 to 2GB of GPU memory.
- Note that this tracker doesn't account for memory allocations outside of [`Trainer`]'s
- `__init__`, `train`, `evaluate` and `predict` calls.
+ Note that this tracker doesn't account for memory allocations outside of [`Trainer`]'s `__init__`, `train`,
+ `evaluate` and `predict` calls.
Because `evaluation` calls may happen during `train`, we can't handle nested invocations because
- `torch.cuda.max_memory_allocated` is a single counter, so if it gets reset by a nested eval call, `train`'s
- tracker will report incorrect info. If this [pytorch issue](https://github.com/pytorch/pytorch/issues/16266)
- gets resolved it will be possible to change this class to be re-entrant. Until then we will only track the outer
- level of `train`, `evaluate` and `predict` methods. Which means that if `eval` is called during `train`,
- it's the latter that will account for its memory usage and that of the former.
+ `torch.cuda.max_memory_allocated` is a single counter, so if it gets reset by a nested eval call, `train`'s tracker
+ will report incorrect info. If this [pytorch issue](https://github.com/pytorch/pytorch/issues/16266) gets resolved
+ it will be possible to change this class to be re-entrant. Until then we will only track the outer level of
+ `train`, `evaluate` and `predict` methods. Which means that if `eval` is called during `train`, it's the latter
+ that will account for its memory usage and that of the former.
This also means that if any other tool that is used along the [`Trainer`] calls
- `torch.cuda.reset_peak_memory_stats`, the gpu peak memory stats could be invalid. And the
- [`Trainer`] will disrupt the normal behavior of any such tools that rely on calling
- `torch.cuda.reset_peak_memory_stats` themselves.
+ `torch.cuda.reset_peak_memory_stats`, the gpu peak memory stats could be invalid. And the [`Trainer`] will disrupt
+ the normal behavior of any such tools that rely on calling `torch.cuda.reset_peak_memory_stats` themselves.
For best performance you may want to consider turning the memory profiling off for production runs.
"""
@@ -959,8 +956,8 @@ def save_metrics(self, split, metrics, combined=True):
combined (`bool`, *optional*, defaults to `True`):
Creates combined metrics by updating `all_results.json` with metrics of this call
- To understand the metrics please read the docstring of [`~Trainer.log_metrics`]. The only
- difference is that raw unformatted numbers are saved in the current method.
+ To understand the metrics please read the docstring of [`~Trainer.log_metrics`]. The only difference is that raw
+ unformatted numbers are saved in the current method.
"""
if not self.is_world_process_zero():
diff --git a/src/transformers/trainer_seq2seq.py b/src/transformers/trainer_seq2seq.py
index a8af7f715f..9f6fb3ecdf 100644
--- a/src/transformers/trainer_seq2seq.py
+++ b/src/transformers/trainer_seq2seq.py
@@ -46,9 +46,9 @@ class Seq2SeqTrainer(Trainer):
Args:
eval_dataset (`Dataset`, *optional*):
- Pass a dataset if you wish to override `self.eval_dataset`. If it is an `datasets.Dataset`,
- columns not accepted by the `model.forward()` method are automatically removed. It must implement the
- `__len__` method.
+ Pass a dataset if you wish to override `self.eval_dataset`. If it is an `datasets.Dataset`, columns not
+ accepted by the `model.forward()` method are automatically removed. It must implement the `__len__`
+ method.
ignore_keys (`List[str]`, *optional*):
A list of keys in the output of your model (if it is a dictionary) that should be ignored when
gathering predictions.
@@ -111,8 +111,8 @@ class Seq2SeqTrainer(Trainer):
- predictions (`np.ndarray`): The predictions on `test_dataset`.
- label_ids (`np.ndarray`, *optional*): The labels (if the dataset contained some).
- - metrics (`Dict[str, float]`, *optional*): The potential dictionary of metrics (if the dataset
- contained labels).
+ - metrics (`Dict[str, float]`, *optional*): The potential dictionary of metrics (if the dataset contained
+ labels).
"""
self._max_length = max_length if max_length is not None else self.args.generation_max_length
self._num_beams = num_beams if num_beams is not None else self.args.generation_num_beams
diff --git a/src/transformers/trainer_tf.py b/src/transformers/trainer_tf.py
index 03591cbb76..9fd8f65567 100644
--- a/src/transformers/trainer_tf.py
+++ b/src/transformers/trainer_tf.py
@@ -58,28 +58,27 @@ class TFTrainer:
args ([`TFTrainingArguments`]):
The arguments to tweak training.
train_dataset ([`~tf.data.Dataset`], *optional*):
- The dataset to use for training. The dataset should yield tuples of `(features, labels)` where
- `features` is a dict of input features and `labels` is the labels. If `labels` is a tensor, the loss
- is calculated by the model by calling `model(features, labels=labels)`. If `labels` is a dict, such as
- when using a QuestionAnswering head model with multiple targets, the loss is instead calculated by calling
+ The dataset to use for training. The dataset should yield tuples of `(features, labels)` where `features`
+ is a dict of input features and `labels` is the labels. If `labels` is a tensor, the loss is calculated by
+ the model by calling `model(features, labels=labels)`. If `labels` is a dict, such as when using a
+ QuestionAnswering head model with multiple targets, the loss is instead calculated by calling
`model(features, **labels)`.
eval_dataset ([`~tf.data.Dataset`], *optional*):
- The dataset to use for evaluation. The dataset should yield tuples of `(features, labels)` where
- `features` is a dict of input features and `labels` is the labels. If `labels` is a tensor, the loss
- is calculated by the model by calling `model(features, labels=labels)`. If `labels` is a dict, such as
- when using a QuestionAnswering head model with multiple targets, the loss is instead calculated by calling
+ The dataset to use for evaluation. The dataset should yield tuples of `(features, labels)` where `features`
+ is a dict of input features and `labels` is the labels. If `labels` is a tensor, the loss is calculated by
+ the model by calling `model(features, labels=labels)`. If `labels` is a dict, such as when using a
+ QuestionAnswering head model with multiple targets, the loss is instead calculated by calling
`model(features, **labels)`.
compute_metrics (`Callable[[EvalPrediction], Dict]`, *optional*):
- The function that will be used to compute metrics at evaluation. Must take a
- [`EvalPrediction`] and return a dictionary string to metric values.
+ The function that will be used to compute metrics at evaluation. Must take a [`EvalPrediction`] and return
+ a dictionary string to metric values.
tb_writer (`tf.summary.SummaryWriter`, *optional*):
Object to write to TensorBoard.
optimizers (`Tuple[tf.keras.optimizers.Optimizer, tf.keras.optimizers.schedules.LearningRateSchedule]`, *optional*):
A tuple containing the optimizer and the scheduler to use. The optimizer default to an instance of
- [`tf.keras.optimizers.Adam`] if `args.weight_decay_rate` is 0 else an instance of
- [`AdamWeightDecay`]. The scheduler will default to an instance of
- [`tf.keras.optimizers.schedules.PolynomialDecay`] if `args.num_warmup_steps` is 0 else an
- instance of [`WarmUp`].
+ [`tf.keras.optimizers.Adam`] if `args.weight_decay_rate` is 0 else an instance of [`AdamWeightDecay`]. The
+ scheduler will default to an instance of [`tf.keras.optimizers.schedules.PolynomialDecay`] if
+ `args.num_warmup_steps` is 0 else an instance of [`WarmUp`].
"""
def __init__(
@@ -167,10 +166,11 @@ class TFTrainer:
Args:
eval_dataset ([`~tf.data.Dataset`], *optional*):
- If provided, will override *self.eval_dataset*. The dataset should yield tuples of `(features, labels)` where `features` is a dict of input features and `labels` is the labels. If `labels` is
- a tensor, the loss is calculated by the model by calling `model(features, labels=labels)`. If
- `labels` is a dict, such as when using a QuestionAnswering head model with multiple targets, the loss
- is instead calculated by calling `model(features, **labels)`.
+ If provided, will override *self.eval_dataset*. The dataset should yield tuples of `(features, labels)`
+ where `features` is a dict of input features and `labels` is the labels. If `labels` is a tensor, the
+ loss is calculated by the model by calling `model(features, labels=labels)`. If `labels` is a dict,
+ such as when using a QuestionAnswering head model with multiple targets, the loss is instead calculated
+ by calling `model(features, **labels)`.
Subclass and override this method if you want to inject some custom behavior.
"""
@@ -199,10 +199,10 @@ class TFTrainer:
Args:
test_dataset ([`~tf.data.Dataset`]):
- The dataset to use. The dataset should yield tuples of `(features, labels)` where `features` is a
- dict of input features and `labels` is the labels. If `labels` is a tensor, the loss is calculated
- by the model by calling `model(features, labels=labels)`. If `labels` is a dict, such as when using
- a QuestionAnswering head model with multiple targets, the loss is instead calculated by calling
+ The dataset to use. The dataset should yield tuples of `(features, labels)` where `features` is a dict
+ of input features and `labels` is the labels. If `labels` is a tensor, the loss is calculated by the
+ model by calling `model(features, labels=labels)`. If `labels` is a dict, such as when using a
+ QuestionAnswering head model with multiple targets, the loss is instead calculated by calling
`model(features, **labels)`.
Subclass and override this method if you want to inject some custom behavior.
@@ -301,8 +301,7 @@ class TFTrainer:
prediction_loss_only: Optional[bool] = None,
) -> PredictionOutput:
"""
- Prediction/evaluation loop, shared by [`~TFTrainer.evaluate`] and
- [`~TFTrainer.predict`].
+ Prediction/evaluation loop, shared by [`~TFTrainer.evaluate`] and [`~TFTrainer.predict`].
Works both with or without labels.
"""
@@ -422,8 +421,9 @@ class TFTrainer:
eval_dataset ([`~tf.data.Dataset`], *optional*):
Pass a dataset if you wish to override `self.eval_dataset`. The dataset should yield tuples of
`(features, labels)` where `features` is a dict of input features and `labels` is the labels. If
- `labels` is a tensor, the loss is calculated by the model by calling `model(features, labels=labels)`. If `labels` is a dict, such as when using a QuestionAnswering head model with
- multiple targets, the loss is instead calculated by calling `model(features, **labels)`.
+ `labels` is a tensor, the loss is calculated by the model by calling `model(features, labels=labels)`.
+ If `labels` is a dict, such as when using a QuestionAnswering head model with multiple targets, the
+ loss is instead calculated by calling `model(features, **labels)`.
Returns:
A dictionary containing the evaluation loss and the potential metrics computed from the predictions.
@@ -767,17 +767,17 @@ class TFTrainer:
Args:
test_dataset ([`~tf.data.Dataset`]):
Dataset to run the predictions on. The dataset should yield tuples of `(features, labels)` where
- `features` is a dict of input features and `labels` is the labels. If `labels` is a tensor, the
- loss is calculated by the model by calling `model(features, labels=labels)`. If `labels` is a dict,
- such as when using a QuestionAnswering head model with multiple targets, the loss is instead calculated
- by calling `model(features, **labels)`
+ `features` is a dict of input features and `labels` is the labels. If `labels` is a tensor, the loss is
+ calculated by the model by calling `model(features, labels=labels)`. If `labels` is a dict, such as
+ when using a QuestionAnswering head model with multiple targets, the loss is instead calculated by
+ calling `model(features, **labels)`
Returns: *NamedTuple* A namedtuple with the following keys:
- predictions (`np.ndarray`): The predictions on `test_dataset`.
- label_ids (`np.ndarray`, *optional*): The labels (if the dataset contained some).
- - metrics (`Dict[str, float]`, *optional*): The potential dictionary of metrics (if the dataset
- contained labels).
+ - metrics (`Dict[str, float]`, *optional*): The potential dictionary of metrics (if the dataset contained
+ labels).
"""
test_ds, steps, num_examples = self.get_test_tfdataset(test_dataset)
diff --git a/src/transformers/trainer_utils.py b/src/transformers/trainer_utils.py
index 3c29b221eb..e822457676 100644
--- a/src/transformers/trainer_utils.py
+++ b/src/transformers/trainer_utils.py
@@ -49,8 +49,7 @@ if is_tf_available():
def set_seed(seed: int):
"""
- Helper function for reproducible behavior to set the seed in `random`, `numpy`, `torch` and/or `tf` (if
- installed).
+ Helper function for reproducible behavior to set the seed in `random`, `numpy`, `torch` and/or `tf` (if installed).
Args:
seed (`int`): The seed to set.
diff --git a/src/transformers/training_args.py b/src/transformers/training_args.py
index 41bd3a179c..6209c742c6 100644
--- a/src/transformers/training_args.py
+++ b/src/transformers/training_args.py
@@ -75,29 +75,29 @@ class TrainingArguments:
TrainingArguments is the subset of the arguments we use in our example scripts **which relate to the training loop
itself**.
- Using [`HfArgumentParser`] we can turn this class into [argparse](https://docs.python.org/3/library/argparse#module-argparse) arguments that can be specified on the command
- line.
+ Using [`HfArgumentParser`] we can turn this class into
+ [argparse](https://docs.python.org/3/library/argparse#module-argparse) arguments that can be specified on the
+ command line.
Parameters:
output_dir (`str`):
The output directory where the model predictions and checkpoints will be written.
overwrite_output_dir (`bool`, *optional*, defaults to `False`):
- If `True`, overwrite the content of the output directory. Use this to continue training if
- `output_dir` points to a checkpoint directory.
+ If `True`, overwrite the content of the output directory. Use this to continue training if `output_dir`
+ points to a checkpoint directory.
do_train (`bool`, *optional*, defaults to `False`):
- Whether to run training or not. This argument is not directly used by [`Trainer`], it's
- intended to be used by your training/evaluation scripts instead. See the [example scripts](https://github.com/huggingface/transformers/tree/master/examples) for more details.
+ Whether to run training or not. This argument is not directly used by [`Trainer`], it's intended to be used
+ by your training/evaluation scripts instead. See the [example
+ scripts](https://github.com/huggingface/transformers/tree/master/examples) for more details.
do_eval (`bool`, *optional*):
- Whether to run evaluation on the validation set or not. Will be set to `True` if
- `evaluation_strategy` is different from `"no"`. This argument is not directly used by
- [`Trainer`], it's intended to be used by your training/evaluation scripts instead. See
- the [example scripts](https://github.com/huggingface/transformers/tree/master/examples) for more
- details.
+ Whether to run evaluation on the validation set or not. Will be set to `True` if `evaluation_strategy` is
+ different from `"no"`. This argument is not directly used by [`Trainer`], it's intended to be used by your
+ training/evaluation scripts instead. See the [example
+ scripts](https://github.com/huggingface/transformers/tree/master/examples) for more details.
do_predict (`bool`, *optional*, defaults to `False`):
- Whether to run predictions on the test set or not. This argument is not directly used by
- [`Trainer`], it's intended to be used by your training/evaluation scripts instead. See
- the [example scripts](https://github.com/huggingface/transformers/tree/master/examples) for more
- details.
+ Whether to run predictions on the test set or not. This argument is not directly used by [`Trainer`], it's
+ intended to be used by your training/evaluation scripts instead. See the [example
+ scripts](https://github.com/huggingface/transformers/tree/master/examples) for more details.
evaluation_strategy (`str` or [`~trainer_utils.IntervalStrategy`], *optional*, defaults to `"no"`):
The evaluation strategy to adopt during training. Possible values are:
@@ -116,9 +116,8 @@ class TrainingArguments:
- When using gradient accumulation, one step is counted as one step with backward pass. Therefore,
- logging, evaluation, save will be conducted every `gradient_accumulation_steps * xxx_step` training
- examples.
+ When using gradient accumulation, one step is counted as one step with backward pass. Therefore, logging,
+ evaluation, save will be conducted every `gradient_accumulation_steps * xxx_step` training examples.
@@ -129,8 +128,8 @@ class TrainingArguments:
learning_rate (`float`, *optional*, defaults to 5e-5):
The initial learning rate for [`AdamW`] optimizer.
weight_decay (`float`, *optional*, defaults to 0):
- The weight decay to apply (if not zero) to all layers except all bias and LayerNorm weights in
- [`AdamW`] optimizer.
+ The weight decay to apply (if not zero) to all layers except all bias and LayerNorm weights in [`AdamW`]
+ optimizer.
adam_beta1 (`float`, *optional*, defaults to 0.9):
The beta1 hyperparameter for the [`AdamW`] optimizer.
adam_beta2 (`float`, *optional*, defaults to 0.999):
@@ -143,17 +142,15 @@ class TrainingArguments:
Total number of training epochs to perform (if not an integer, will perform the decimal part percents of
the last epoch before stopping training).
max_steps (`int`, *optional*, defaults to -1):
- If set to a positive number, the total number of training steps to perform. Overrides
- `num_train_epochs`. In case of using a finite iterable dataset the training may stop before reaching
- the set number of steps when all data is exhausted
+ If set to a positive number, the total number of training steps to perform. Overrides `num_train_epochs`.
+ In case of using a finite iterable dataset the training may stop before reaching the set number of steps
+ when all data is exhausted
lr_scheduler_type (`str` or [`SchedulerType`], *optional*, defaults to `"linear"`):
- The scheduler type to use. See the documentation of [`SchedulerType`] for all possible
- values.
+ The scheduler type to use. See the documentation of [`SchedulerType`] for all possible values.
warmup_ratio (`float`, *optional*, defaults to 0.0):
Ratio of total training steps used for a linear warmup from 0 to `learning_rate`.
warmup_steps (`int`, *optional*, defaults to 0):
- Number of steps used for a linear warmup from 0 to `learning_rate`. Overrides any effect of
- `warmup_ratio`.
+ Number of steps used for a linear warmup from 0 to `learning_rate`. Overrides any effect of `warmup_ratio`.
log_level (`str`, *optional*, defaults to `passive`):
Logger log level to use on the main process. Possible choices are the log levels as strings: 'debug',
'info', 'warning', 'error' and 'critical', plus a 'passive' level which doesn't set anything and lets the
@@ -178,14 +175,13 @@ class TrainingArguments:
logging_steps (`int`, *optional*, defaults to 500):
Number of update steps between two logs if `logging_strategy="steps"`.
logging_nan_inf_filter (`bool`, *optional*, defaults to `True`):
- Whether to filter `nan` and `inf` losses for logging. If set to obj:*True* the loss of every step
- that is `nan` or `inf` is filtered and the average loss of the current logging window is taken
- instead.
+ Whether to filter `nan` and `inf` losses for logging. If set to obj:*True* the loss of every step that is
+ `nan` or `inf` is filtered and the average loss of the current logging window is taken instead.
- `logging_nan_inf_filter` only influences the logging of loss values, it does not change the
- behavior the gradient is computed or applied to the model.
+ `logging_nan_inf_filter` only influences the logging of loss values, it does not change the behavior the
+ gradient is computed or applied to the model.
@@ -210,22 +206,21 @@ class TrainingArguments:
Whether to not use CUDA even when it is available or not.
seed (`int`, *optional*, defaults to 42):
Random seed that will be set at the beginning of training. To ensure reproducibility across runs, use the
- [`~Trainer.model_init`] function to instantiate the model if it has some randomly
- initialized parameters.
+ [`~Trainer.model_init`] function to instantiate the model if it has some randomly initialized parameters.
bf16 (`bool`, *optional*, defaults to `False`):
Whether to use bf16 16-bit (mixed) precision training instead of 32-bit training. Requires Ampere or higher
NVIDIA architecture. This is an experimental API and it may change.
fp16 (`bool`, *optional*, defaults to `False`):
Whether to use fp16 16-bit (mixed) precision training instead of 32-bit training.
fp16_opt_level (`str`, *optional*, defaults to 'O1'):
- For `fp16` training, Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']. See details
- on the [Apex documentation](https://nvidia.github.io/apex/amp).
+ For `fp16` training, Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']. See details on
+ the [Apex documentation](https://nvidia.github.io/apex/amp).
fp16_backend (`str`, *optional*, defaults to `"auto"`):
This argument is deprecated. Use `half_precision_backend` instead.
half_precision_backend (`str`, *optional*, defaults to `"auto"`):
- The backend to use for mixed precision training. Must be one of `"auto"`, `"amp"` or
- `"apex"`. `"auto"` will use AMP or APEX depending on the PyTorch version detected, while the
- other choices will force the requested backend.
+ The backend to use for mixed precision training. Must be one of `"auto"`, `"amp"` or `"apex"`. `"auto"`
+ will use AMP or APEX depending on the PyTorch version detected, while the other choices will force the
+ requested backend.
bf16_full_eval (`bool`, *optional*, defaults to `False`):
Whether to use full bfloat16 evaluation instead of 32-bit. This will be faster and save memory but can harm
metric values. This is an experimental API and it may change.
@@ -245,22 +240,23 @@ class TrainingArguments:
Whether to drop the last incomplete batch (if the length of the dataset is not divisible by the batch size)
or not.
eval_steps (`int`, *optional*):
- Number of update steps between two evaluations if `evaluation_strategy="steps"`. Will default to the
- same value as `logging_steps` if not set.
+ Number of update steps between two evaluations if `evaluation_strategy="steps"`. Will default to the same
+ value as `logging_steps` if not set.
dataloader_num_workers (`int`, *optional*, defaults to 0):
Number of subprocesses to use for data loading (PyTorch only). 0 means that the data will be loaded in the
main process.
past_index (`int`, *optional*, defaults to -1):
- Some models like [TransformerXL](../model_doc/transformerxl) or [XLNet](../model_doc/xlnet) can
- make use of the past hidden states for their predictions. If this argument is set to a positive int, the
- `Trainer` will use the corresponding output (usually index 2) as the past state and feed it to the model
- at the next training step under the keyword argument `mems`.
+ Some models like [TransformerXL](../model_doc/transformerxl) or [XLNet](../model_doc/xlnet) can make use of
+ the past hidden states for their predictions. If this argument is set to a positive int, the `Trainer` will
+ use the corresponding output (usually index 2) as the past state and feed it to the model at the next
+ training step under the keyword argument `mems`.
run_name (`str`, *optional*):
- A descriptor for the run. Typically used for [wandb](https://www.wandb.com/) and [mlflow](https://www.mlflow.org/) logging.
+ A descriptor for the run. Typically used for [wandb](https://www.wandb.com/) and
+ [mlflow](https://www.mlflow.org/) logging.
disable_tqdm (`bool`, *optional*):
Whether or not to disable the tqdm progress bars and table of metrics produced by
- [`~notebook.NotebookTrainingTracker`] in Jupyter Notebooks. Will default to `True`
- if the logging level is set to warn or lower (default), `False` otherwise.
+ [`~notebook.NotebookTrainingTracker`] in Jupyter Notebooks. Will default to `True` if the logging level is
+ set to warn or lower (default), `False` otherwise.
remove_unused_columns (`bool`, *optional*, defaults to `True`):
If using `datasets.Dataset` datasets, whether or not to automatically remove the columns unused by the
model forward method.
@@ -269,50 +265,46 @@ class TrainingArguments:
label_names (`List[str]`, *optional*):
The list of keys in your dictionary of inputs that correspond to the labels.
- Will eventually default to `["labels"]` except if the model used is one of the
- `XxxForQuestionAnswering` in which case it will default to `["start_positions", "end_positions"]`.
+ Will eventually default to `["labels"]` except if the model used is one of the `XxxForQuestionAnswering` in
+ which case it will default to `["start_positions", "end_positions"]`.
load_best_model_at_end (`bool`, *optional*, defaults to `False`):
Whether or not to load the best model found during training at the end of training.
- When set to `True`, the parameters `save_strategy` needs to be the same as
- `eval_strategy`, and in the case it is "steps", `save_steps` must be a round multiple of
- `eval_steps`.
+ When set to `True`, the parameters `save_strategy` needs to be the same as `eval_strategy`, and in the case
+ it is "steps", `save_steps` must be a round multiple of `eval_steps`.
metric_for_best_model (`str`, *optional*):
Use in conjunction with `load_best_model_at_end` to specify the metric to use to compare two different
- models. Must be the name of a metric returned by the evaluation with or without the prefix `"eval_"`.
- Will default to `"loss"` if unspecified and `load_best_model_at_end=True` (to use the evaluation
- loss).
+ models. Must be the name of a metric returned by the evaluation with or without the prefix `"eval_"`. Will
+ default to `"loss"` if unspecified and `load_best_model_at_end=True` (to use the evaluation loss).
- If you set this value, `greater_is_better` will default to `True`. Don't forget to set it to
- `False` if your metric is better when lower.
+ If you set this value, `greater_is_better` will default to `True`. Don't forget to set it to `False` if
+ your metric is better when lower.
greater_is_better (`bool`, *optional*):
- Use in conjunction with `load_best_model_at_end` and `metric_for_best_model` to specify if better
- models should have a greater metric or not. Will default to:
+ Use in conjunction with `load_best_model_at_end` and `metric_for_best_model` to specify if better models
+ should have a greater metric or not. Will default to:
- - `True` if `metric_for_best_model` is set to a value that isn't `"loss"` or
- `"eval_loss"`.
+ - `True` if `metric_for_best_model` is set to a value that isn't `"loss"` or `"eval_loss"`.
- `False` if `metric_for_best_model` is not set, or set to `"loss"` or `"eval_loss"`.
ignore_data_skip (`bool`, *optional*, defaults to `False`):
When resuming training, whether or not to skip the epochs and batches to get the data loading at the same
- stage as in the previous training. If set to `True`, the training will begin faster (as that skipping
- step can take a long time) but will not yield the same results as the interrupted training would have.
+ stage as in the previous training. If set to `True`, the training will begin faster (as that skipping step
+ can take a long time) but will not yield the same results as the interrupted training would have.
sharded_ddp (`bool`, `str` or list of [`~trainer_utils.ShardedDDPOption`], *optional*, defaults to `False`):
Use Sharded DDP training from [FairScale](https://github.com/facebookresearch/fairscale) (in distributed
training only). This is an experimental feature.
A list of options along the following:
- - `"simple"`: to use first instance of sharded DDP released by fairscale (`ShardedDDP`) similar
- to ZeRO-2.
- - `"zero_dp_2"`: to use the second instance of sharded DPP released by fairscale
- (`FullyShardedDDP`) in Zero-2 mode (with `reshard_after_forward=False`).
- - `"zero_dp_3"`: to use the second instance of sharded DPP released by fairscale
- (`FullyShardedDDP`) in Zero-3 mode (with `reshard_after_forward=True`).
+ - `"simple"`: to use first instance of sharded DDP released by fairscale (`ShardedDDP`) similar to ZeRO-2.
+ - `"zero_dp_2"`: to use the second instance of sharded DPP released by fairscale (`FullyShardedDDP`) in
+ Zero-2 mode (with `reshard_after_forward=False`).
+ - `"zero_dp_3"`: to use the second instance of sharded DPP released by fairscale (`FullyShardedDDP`) in
+ Zero-3 mode (with `reshard_after_forward=True`).
- `"offload"`: to add ZeRO-offload (only compatible with `"zero_dp_2"` and `"zero_dp_3"`).
If a string is passed, it will be split on space. If a bool is passed, it will be converted to an empty
@@ -323,38 +315,36 @@ class TrainingArguments:
`ds_config.json`) or an already loaded json file as a `dict`"
label_smoothing_factor (`float`, *optional*, defaults to 0.0):
The label smoothing factor to use. Zero means no label smoothing, otherwise the underlying onehot-encoded
- labels are changed from 0s and 1s to `label_smoothing_factor/num_labels` and `1 - label_smoothing_factor + label_smoothing_factor/num_labels` respectively.
+ labels are changed from 0s and 1s to `label_smoothing_factor/num_labels` and `1 - label_smoothing_factor +
+ label_smoothing_factor/num_labels` respectively.
debug (`str` or list of [`~debug_utils.DebugOption`], *optional*, defaults to `""`):
Enable one or more debug features. This is an experimental feature.
Possible options are:
- - `"underflow_overflow"`: detects overflow in model's input/outputs and reports the last frames that
- led to the event
+ - `"underflow_overflow"`: detects overflow in model's input/outputs and reports the last frames that led to
+ the event
- `"tpu_metrics_debug"`: print debug metrics on TPU
The options should be separated by whitespaces.
adafactor (`bool`, *optional*, defaults to `False`):
- Whether or not to use the [`Adafactor`] optimizer instead of
- [`AdamW`].
+ Whether or not to use the [`Adafactor`] optimizer instead of [`AdamW`].
group_by_length (`bool`, *optional*, defaults to `False`):
Whether or not to group together samples of roughly the same length in the training dataset (to minimize
padding applied and be more efficient). Only useful if applying dynamic padding.
length_column_name (`str`, *optional*, defaults to `"length"`):
Column name for precomputed lengths. If the column exists, grouping by length will use these values rather
- than computing them on train startup. Ignored unless `group_by_length` is `True` and the dataset
- is an instance of `Dataset`.
+ than computing them on train startup. Ignored unless `group_by_length` is `True` and the dataset is an
+ instance of `Dataset`.
report_to (`str` or `List[str]`, *optional*, defaults to `"all"`):
The list of integrations to report the results and logs to. Supported platforms are `"azure_ml"`,
- `"comet_ml"`, `"mlflow"`, `"tensorboard"` and `"wandb"`. Use `"all"` to report to
- all integrations installed, `"none"` for no integrations.
+ `"comet_ml"`, `"mlflow"`, `"tensorboard"` and `"wandb"`. Use `"all"` to report to all integrations
+ installed, `"none"` for no integrations.
ddp_find_unused_parameters (`bool`, *optional*):
When using distributed training, the value of the flag `find_unused_parameters` passed to
- `DistributedDataParallel`. Will default to `False` if gradient checkpointing is used, `True`
- otherwise.
+ `DistributedDataParallel`. Will default to `False` if gradient checkpointing is used, `True` otherwise.
ddp_bucket_cap_mb (`int`, *optional*):
- When using distributed training, the value of the flag `bucket_cap_mb` passed to
- `DistributedDataParallel`.
+ When using distributed training, the value of the flag `bucket_cap_mb` passed to `DistributedDataParallel`.
dataloader_pin_memory (`bool`, *optional*, defaults to `True`):
Whether you want to pin memory in data loaders or not. Will default to `True`.
skip_memory_metrics (`bool`, *optional*, defaults to `True`):
@@ -362,36 +352,34 @@ class TrainingArguments:
down the training and evaluation speed.
push_to_hub (`bool`, *optional*, defaults to `False`):
Whether or not to upload the trained model to the hub after training. If this is activated, and
- `output_dir` exists, it needs to be a local clone of the repository to which the
- [`Trainer`] will be pushed.
+ `output_dir` exists, it needs to be a local clone of the repository to which the [`Trainer`] will be
+ pushed.
resume_from_checkpoint (`str`, *optional*):
The path to a folder with a valid checkpoint for your model. This argument is not directly used by
- [`Trainer`], it's intended to be used by your training/evaluation scripts instead. See
- the [example scripts](https://github.com/huggingface/transformers/tree/master/examples) for more
- details.
+ [`Trainer`], it's intended to be used by your training/evaluation scripts instead. See the [example
+ scripts](https://github.com/huggingface/transformers/tree/master/examples) for more details.
hub_model_id (`str`, *optional*):
The name of the repository to keep in sync with the local *output_dir*. It can be a simple model ID in
which case the model will be pushed in your namespace. Otherwise it should be the whole repository name,
for instance `"user_name/model"`, which allows you to push to an organization you are a member of with
- `"organization_name/model"`. Will default to `user_name/output_dir_name` with *output_dir_name*
- being the name of `output_dir`.
+ `"organization_name/model"`. Will default to `user_name/output_dir_name` with *output_dir_name* being the
+ name of `output_dir`.
Will default to to the name of `output_dir`.
hub_strategy (`str` or [`~trainer_utils.HubStrategy`], *optional*, defaults to `"every_save"`):
Defines the scope of what is pushed to the Hub and when. Possible values are:
- - `"end"`: push the model, its configuration, the tokenizer (if passed along to the
- [`Trainer`]) and a draft of a model card at the end of training.
- - `"every_save"`: push the model, its configuration, the tokenizer (if passed along to the
- [`Trainer`]) and a draft of a model card each time there is a model save. The pushes
- are asynchronous to not block training, and in case the save are very frequent, a new push is only
- attempted if the previous one is finished. A last push is made with the final model at the end of
- training.
- - `"checkpoint"`: like `"every_save"` but the latest checkpoint is also pushed in a subfolder
- named last-checkpoint, allowing you to resume training easily with
+ - `"end"`: push the model, its configuration, the tokenizer (if passed along to the [`Trainer`]) and a
+ draft of a model card at the end of training.
+ - `"every_save"`: push the model, its configuration, the tokenizer (if passed along to the [`Trainer`]) and
+ a draft of a model card each time there is a model save. The pushes are asynchronous to not block
+ training, and in case the save are very frequent, a new push is only attempted if the previous one is
+ finished. A last push is made with the final model at the end of training.
+ - `"checkpoint"`: like `"every_save"` but the latest checkpoint is also pushed in a subfolder named
+ last-checkpoint, allowing you to resume training easily with
`trainer.train(resume_from_checkpoint="last-checkpoint")`.
- - `"all_checkpoints"`: like `"checkpoint"` but all checkpoints are pushed like they appear in the
- output folder (so you will get one checkpoint folder per folder in your final repository)
+ - `"all_checkpoints"`: like `"checkpoint"` but all checkpoints are pushed like they appear in the output
+ folder (so you will get one checkpoint folder per folder in your final repository)
hub_token (`str`, *optional*):
The token to use to push the model to the Hub. Will default to the token in the cache folder obtained with
@@ -1172,11 +1160,10 @@ class TrainingArguments:
For the main process the log level defaults to `logging.INFO` unless overridden by `log_level` argument.
- For the replica processes the log level defaults to `logging.WARNING` unless overridden by
- `log_level_replica` argument.
+ For the replica processes the log level defaults to `logging.WARNING` unless overridden by `log_level_replica`
+ argument.
- The choice between the main and replica process settings is made according to the return value of
- `should_log`.
+ The choice between the main and replica process settings is made according to the return value of `should_log`.
"""
log_level_main_node = logging.INFO if self.log_level == -1 else self.log_level
@@ -1200,17 +1187,17 @@ class TrainingArguments:
@contextlib.contextmanager
def main_process_first(self, local=True, desc="work"):
"""
- A context manager for torch distributed environment where on needs to do something on the main process,
- while blocking replicas, and when it's finished releasing the replicas.
+ A context manager for torch distributed environment where on needs to do something on the main process, while
+ blocking replicas, and when it's finished releasing the replicas.
- One such use is for `datasets`'s `map` feature which to be efficient should be run once on the main
- process, which upon completion saves a cached version of results and which then automatically gets loaded
- by the replicas.
+ One such use is for `datasets`'s `map` feature which to be efficient should be run once on the main process,
+ which upon completion saves a cached version of results and which then automatically gets loaded by the
+ replicas.
Args:
local (`bool`, *optional*, defaults to `True`):
- if `True` first means process of rank 0 of each node if `False` first means process of rank 0
- of node rank 0 In multi-node environment with a shared filesystem you most likely will want to use
+ if `True` first means process of rank 0 of each node if `False` first means process of rank 0 of node
+ rank 0 In multi-node environment with a shared filesystem you most likely will want to use
`local=False` so that only the main process of the first node will do the processing. If however, the
filesystem is not shared, then the main process of each node will need to do the processing, which is
the default behavior.
diff --git a/src/transformers/training_args_seq2seq.py b/src/transformers/training_args_seq2seq.py
index 4f2154d86e..7dcf6a4d56 100644
--- a/src/transformers/training_args_seq2seq.py
+++ b/src/transformers/training_args_seq2seq.py
@@ -36,8 +36,8 @@ class Seq2SeqTrainingArguments(TrainingArguments):
predict_with_generate (`bool`, *optional*, defaults to `False`):
Whether to use generate to calculate generative metrics (ROUGE, BLEU).
generation_max_length (`int`, *optional*):
- The `max_length` to use on each evaluation loop when `predict_with_generate=True`. Will default to
- the `max_length` value of the model configuration.
+ The `max_length` to use on each evaluation loop when `predict_with_generate=True`. Will default to the
+ `max_length` value of the model configuration.
generation_num_beams (`int`, *optional*):
The `num_beams` to use on each evaluation loop when `predict_with_generate=True`. Will default to the
`num_beams` value of the model configuration.
diff --git a/src/transformers/training_args_tf.py b/src/transformers/training_args_tf.py
index 64776525f1..27389c1c7a 100644
--- a/src/transformers/training_args_tf.py
+++ b/src/transformers/training_args_tf.py
@@ -33,29 +33,29 @@ class TFTrainingArguments(TrainingArguments):
TrainingArguments is the subset of the arguments we use in our example scripts **which relate to the training loop
itself**.
- Using [`HfArgumentParser`] we can turn this class into [argparse](https://docs.python.org/3/library/argparse#module-argparse) arguments that can be specified on the command
- line.
+ Using [`HfArgumentParser`] we can turn this class into
+ [argparse](https://docs.python.org/3/library/argparse#module-argparse) arguments that can be specified on the
+ command line.
Parameters:
output_dir (`str`):
The output directory where the model predictions and checkpoints will be written.
overwrite_output_dir (`bool`, *optional*, defaults to `False`):
- If `True`, overwrite the content of the output directory. Use this to continue training if
- `output_dir` points to a checkpoint directory.
+ If `True`, overwrite the content of the output directory. Use this to continue training if `output_dir`
+ points to a checkpoint directory.
do_train (`bool`, *optional*, defaults to `False`):
- Whether to run training or not. This argument is not directly used by [`Trainer`], it's
- intended to be used by your training/evaluation scripts instead. See the [example scripts](https://github.com/huggingface/transformers/tree/master/examples) for more details.
+ Whether to run training or not. This argument is not directly used by [`Trainer`], it's intended to be used
+ by your training/evaluation scripts instead. See the [example
+ scripts](https://github.com/huggingface/transformers/tree/master/examples) for more details.
do_eval (`bool`, *optional*):
- Whether to run evaluation on the validation set or not. Will be set to `True` if
- `evaluation_strategy` is different from `"no"`. This argument is not directly used by
- [`Trainer`], it's intended to be used by your training/evaluation scripts instead. See
- the [example scripts](https://github.com/huggingface/transformers/tree/master/examples) for more
- details.
+ Whether to run evaluation on the validation set or not. Will be set to `True` if `evaluation_strategy` is
+ different from `"no"`. This argument is not directly used by [`Trainer`], it's intended to be used by your
+ training/evaluation scripts instead. See the [example
+ scripts](https://github.com/huggingface/transformers/tree/master/examples) for more details.
do_predict (`bool`, *optional*, defaults to `False`):
- Whether to run predictions on the test set or not. This argument is not directly used by
- [`Trainer`], it's intended to be used by your training/evaluation scripts instead. See
- the [example scripts](https://github.com/huggingface/transformers/tree/master/examples) for more
- details.
+ Whether to run predictions on the test set or not. This argument is not directly used by [`Trainer`], it's
+ intended to be used by your training/evaluation scripts instead. See the [example
+ scripts](https://github.com/huggingface/transformers/tree/master/examples) for more details.
evaluation_strategy (`str` or [`~trainer_utils.IntervalStrategy`], *optional*, defaults to `"no"`):
The evaluation strategy to adopt during training. Possible values are:
@@ -72,9 +72,8 @@ class TFTrainingArguments(TrainingArguments):
- When using gradient accumulation, one step is counted as one step with backward pass. Therefore,
- logging, evaluation, save will be conducted every `gradient_accumulation_steps * xxx_step` training
- examples.
+ When using gradient accumulation, one step is counted as one step with backward pass. Therefore, logging,
+ evaluation, save will be conducted every `gradient_accumulation_steps * xxx_step` training examples.
@@ -93,13 +92,11 @@ class TFTrainingArguments(TrainingArguments):
num_train_epochs(`float`, *optional*, defaults to 3.0):
Total number of training epochs to perform.
max_steps (`int`, *optional*, defaults to -1):
- If set to a positive number, the total number of training steps to perform. Overrides
- `num_train_epochs`.
+ If set to a positive number, the total number of training steps to perform. Overrides `num_train_epochs`.
warmup_ratio (`float`, *optional*, defaults to 0.0):
Ratio of total training steps used for a linear warmup from 0 to `learning_rate`.
warmup_steps (`int`, *optional*, defaults to 0):
- Number of steps used for a linear warmup from 0 to `learning_rate`. Overrides any effect of
- `warmup_ratio`.
+ Number of steps used for a linear warmup from 0 to `learning_rate`. Overrides any effect of `warmup_ratio`.
logging_dir (`str`, *optional*):
[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to
*runs/**CURRENT_DATETIME_HOSTNAME***.
@@ -133,8 +130,8 @@ class TFTrainingArguments(TrainingArguments):
fp16 (`bool`, *optional*, defaults to `False`):
Whether to use 16-bit (mixed) precision training (through NVIDIA Apex) instead of 32-bit training.
fp16_opt_level (`str`, *optional*, defaults to 'O1'):
- For `fp16` training, Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']. See details
- on the [Apex documentation](https://nvidia.github.io/apex/amp).
+ For `fp16` training, Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']. See details on
+ the [Apex documentation](https://nvidia.github.io/apex/amp).
local_rank (`int`, *optional*, defaults to -1):
During distributed training, the rank of the process.
tpu_num_cores (`int`, *optional*):
@@ -147,10 +144,10 @@ class TFTrainingArguments(TrainingArguments):
eval_steps (`int`, *optional*, defaults to 1000):
Number of update steps before two evaluations.
past_index (`int`, *optional*, defaults to -1):
- Some models like [TransformerXL](../model_doc/transformerxl) or :doc*XLNet <../model_doc/xlnet>* can
- make use of the past hidden states for their predictions. If this argument is set to a positive int, the
- `Trainer` will use the corresponding output (usually index 2) as the past state and feed it to the model
- at the next training step under the keyword argument `mems`.
+ Some models like [TransformerXL](../model_doc/transformerxl) or :doc*XLNet <../model_doc/xlnet>* can make
+ use of the past hidden states for their predictions. If this argument is set to a positive int, the
+ `Trainer` will use the corresponding output (usually index 2) as the past state and feed it to the model at
+ the next training step under the keyword argument `mems`.
tpu_name (`str`, *optional*):
The name of the TPU the process is running on.
tpu_zone (`str`, *optional*):
diff --git a/src/transformers/utils/fx.py b/src/transformers/utils/fx.py
index 890a9b5a05..25a2138589 100644
--- a/src/transformers/utils/fx.py
+++ b/src/transformers/utils/fx.py
@@ -404,9 +404,9 @@ class HFTracer(Tracer):
def path_of_module(self, mod: nn.Module) -> str:
"""
- Helper method to find the qualified name of `mod` in the Module hierarchy of `root`. For example, if
- `root` has a submodule named `foo`, which has a submodule named `bar`, passing `bar` into this function
- will return the string "foo.bar".
+ Helper method to find the qualified name of `mod` in the Module hierarchy of `root`. For example, if `root` has
+ a submodule named `foo`, which has a submodule named `bar`, passing `bar` into this function will return the
+ string "foo.bar".
Args:
mod (str): The `Module` to retrieve the qualified name for.
@@ -514,7 +514,8 @@ def symbolic_trace(
The batch size of the traced model inputs.
sequence_length (`int` or `List[int]]`):
The sequence length of the traced model inputs. For sequence-to-sequence models with different sequence
- lengths between the encoder and the decoder inputs, this must be `[encoder_sequence_length, decoder_sequence_length]`.
+ lengths between the encoder and the decoder inputs, this must be `[encoder_sequence_length,
+ decoder_sequence_length]`.
num_choices (`int`, *optional*, defaults to -1):
The number of possible choices for a multiple choice task.
diff --git a/src/transformers/utils/logging.py b/src/transformers/utils/logging.py
index a9046a5455..659022a009 100644
--- a/src/transformers/utils/logging.py
+++ b/src/transformers/utils/logging.py
@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-""" Logging utilities. """
+""" Logging utilities."""
import logging
import os
diff --git a/src/transformers/utils/notebook.py b/src/transformers/utils/notebook.py
index 498e1ffe42..779446f5f1 100644
--- a/src/transformers/utils/notebook.py
+++ b/src/transformers/utils/notebook.py
@@ -45,7 +45,7 @@ def html_progress_bar(value, total, prefix, label, width=300):
def text_to_html_table(items):
"Put the texts in `items` in an HTML table."
html_code = """\n"""
- html_code += """ \n \n"""
+ html_code += """ \n \n"""
for i in items[0]:
html_code += f" | {i} | \n"
html_code += "
\n \n
\n"
@@ -65,11 +65,10 @@ class NotebookProgressBar:
Class attributes (overridden by derived classes)
- - **warmup** (`int`) -- The number of iterations to do at the beginning while ignoring
- `update_every`.
+ - **warmup** (`int`) -- The number of iterations to do at the beginning while ignoring `update_every`.
- **update_every** (`float`) -- Since calling the time takes some time, we only do it every presumed
- `update_every` seconds. The progress bar uses the average time passed up until now to guess the next
- value for which it will call the update.
+ `update_every` seconds. The progress bar uses the average time passed up until now to guess the next value
+ for which it will call the update.
Args:
total (`int`):
@@ -80,8 +79,8 @@ class NotebookProgressBar:
Whether or not to leave the progress bar once it's completed. You can always call the
[`~utils.notebook.NotebookProgressBar.close`] method to make the bar disappear.
parent ([`~notebook.NotebookTrainingTracker`], *optional*):
- A parent object (like [`~utils.notebook.NotebookTrainingTracker`]) that spawns progress
- bars and handle their display. If set, the object passed must have a `display()` method.
+ A parent object (like [`~utils.notebook.NotebookTrainingTracker`]) that spawns progress bars and handle
+ their display. If set, the object passed must have a `display()` method.
width (`int`, *optional*, defaults to 300):
The width (in pixels) that the bar will take.
@@ -127,8 +126,8 @@ class NotebookProgressBar:
The value to use. Must be between 0 and `total`.
force_update (`bool`, *optional*, defaults to `False`):
Whether or not to force and update of the internal state and display (by default, the bar will wait for
- `value` to reach the value it predicted corresponds to a time of more than the `update_every`
- attribute since the last update to avoid adding boilerplate).
+ `value` to reach the value it predicted corresponds to a time of more than the `update_every` attribute
+ since the last update to avoid adding boilerplate).
comment (`str`, *optional*):
A comment to add on the left of the progress bar.
"""
@@ -204,8 +203,7 @@ class NotebookTrainingTracker(NotebookProgressBar):
Args:
- num_steps (`int`): The number of steps during training.
- column_names (`List[str]`, *optional*):
+ num_steps (`int`): The number of steps during training. column_names (`List[str]`, *optional*):
The list of column names for the metrics table (will be inferred from the first call to
[`~utils.notebook.NotebookTrainingTracker.write_line`] if not set).
"""
@@ -268,8 +266,8 @@ class NotebookTrainingTracker(NotebookProgressBar):
class NotebookProgressCallback(TrainerCallback):
"""
- A [`TrainerCallback`] that displays the progress of training or evaluation, optimized for
- Jupyter Notebooks or Google colab.
+ A [`TrainerCallback`] that displays the progress of training or evaluation, optimized for Jupyter Notebooks or
+ Google colab.
"""
def __init__(self):
diff --git a/utils/style_doc.py b/utils/style_doc.py
index 85113a9fd0..73703d1d16 100644
--- a/utils/style_doc.py
+++ b/utils/style_doc.py
@@ -18,72 +18,59 @@ import argparse
import os
import re
import warnings
-from enum import Enum
-# Special blocks where the inside should be formatted.
-TEXTUAL_BLOCKS = ["note", "warning"]
-# List of acceptable characters for titles and sections underline.
-TITLE_SPECIAL_CHARS = """= - ` : ' " ~ ^ _ * + # < >""".split(" ")
-# Special words for docstrings (s? means the s is optional)
-DOC_SPECIAL_WORD = [
- "Args?",
- "Params?",
- "Parameters?",
- "Arguments?",
- "Examples?",
- "Usage",
- "Returns?",
- "Raises?",
- "Attributes?",
-]
-
# Regexes
-# Matches any declaration of textual block, like `.. note::`. (ignore case to avoid writing all versions in the list)
-_re_textual_blocks = re.compile(r"^\s*\.\.\s+(" + "|".join(TEXTUAL_BLOCKS) + r")\s*::\s*$", re.IGNORECASE)
-# Matches list introduction in rst.
+# Re pattern that catches list introduction (with potential indent)
_re_list = re.compile(r"^(\s*-\s+|\s*\*\s+|\s*\d+\.\s+)")
-# Matches the indent in a line.
-_re_indent = re.compile(r"^(\s*)\S")
-# Matches a table declaration in rst.
-_re_table = re.compile(r"(\+-+)+\+\s*$")
-# Matches a code block in rst `:: `.
-_re_code_block = re.compile(r"^\s*::\s*$")
-_re_code_block_explicit = re.compile(r"^\.\.\s+code\-block::")
-# Matches any block of the form `.. something::` or `.. something:: bla`.
-_re_ignore = re.compile(r"^\s*\.\.\s+(.*?)\s*::\s*\S*\s*$")
-# Matches comment introduction in rst.
-_re_comment = re.compile(r"\s*\.\.\s*$")
+# Re pattern that catches code block introduction (with potentinal indent)
+_re_code = re.compile(r"^(\s*)```(.*)$")
+# Re pattern that catches rst args blocks of the form `Parameters:`.
+_re_args = re.compile("^\s*(Args?|Arguments?|Params?|Parameters?):\s*$")
+# Re pattern that catches return blocks of the form `Return:`.
+_re_returns = re.compile("^\s*Returns?:\s*$")
# Matches the special tag to ignore some paragraphs.
_re_doc_ignore = re.compile(r"(\.\.|#)\s*docstyle-ignore")
-# Matches the example introduction in docstrings.
-_re_example = re.compile(r"::\s*$")
-# Matches the parameters introduction in docstrings.
-_re_arg_def = re.compile(r"^\s*(Args?|Parameters?|Params|Arguments?|Environment|Attributes?)\s*:\s*$")
-# Matches the return introduction in docstrings.
-_re_return = re.compile(r"^\s*(Returns?|Raises?|Note)\s*:\s*$")
-# Matches any doc special word.
-_re_any_doc_special_word = re.compile(r"^\s*(" + "|".join(DOC_SPECIAL_WORD) + r")::?\s*$")
-class SpecialBlock(Enum):
- NOT_SPECIAL = 0
- NO_STYLE = 1
- ARG_LIST = 2
+DOCTEST_PROMPTS = [">>>", "..."]
-def split_text_in_lines(text, max_len, prefix="", min_indent=None):
+def is_empty_line(line):
+ return len(line) == 0 or line.isspace()
+
+
+def find_indent(line):
"""
- Split `text` in the biggest lines possible with the constraint of `max_len` using `prefix` on the first line and
- then indenting with the same length as `prefix`.
+ Returns the number of spaces that start a line indent.
+ """
+ search = re.search("^(\s*)(?:\S|$)", line)
+ if search is None:
+ return 0
+ return len(search.groups()[0])
+
+
+def format_text(text, max_len, prefix="", min_indent=None):
+ """
+ Format a text in the biggest lines possible with the constraint of a maximum length and an indentation.
+
+ Args:
+ text (`str`): The text to format
+ max_len (`int`): The maximum length per line to use
+ prefix (`str`, *optional*, defaults to `""`): A prefix that will be added to the text.
+ The prefix doesn't count toward the indent (like a - introducing a list).
+ min_indent (`int`, *optional*): The minimum indent of the text.
+ If not set, will default to the length of the `prefix`.
+
+ Returns:
+ `str`: The formatted text.
"""
text = re.sub(r"\s+", " ", text)
- indent = " " * len(prefix)
if min_indent is not None:
- if len(indent) < len(min_indent):
- indent = min_indent
- if len(prefix) < len(min_indent):
- prefix = " " * (len(min_indent) - len(prefix)) + prefix
+ if len(prefix) < min_indent:
+ prefix = " " * (min_indent - len(prefix)) + prefix
+
+ indent = " " * len(prefix)
new_lines = []
words = text.split(" ")
current_line = f"{prefix}{words[0]}"
@@ -98,395 +85,147 @@ def split_text_in_lines(text, max_len, prefix="", min_indent=None):
return "\n".join(new_lines)
-def get_indent(line):
- """Get the indentation of `line`."""
- indent_search = _re_indent.search(line)
- return indent_search.groups()[0] if indent_search is not None else ""
+def split_line_on_first_colon(line):
+ splits = line.split(":")
+ return splits[0], ":".join(splits[1:])
-class CodeStyler:
- """A generic class to style .rst files."""
+def style_docstring(docstring, max_len):
+ """
+ Style a docstring by making sure there is no useless whitespace and the maximum horizontal space is used.
- def is_no_style_block(self, line):
- """Whether or not `line` introduces a block where styling should be ignore"""
- if _re_code_block.search(line) is not None:
- return True
- if _re_textual_blocks.search(line) is not None:
- return False
- return _re_ignore.search(line) is not None
+ Args:
+ docstring (`str`): The docstring to style.
+ max_len (`int`): The maximum length of each line.
- def is_comment_or_textual_block(self, line):
- """Whether or not `line` introduces a block where styling should not be ignored (note, warnings...)"""
- if _re_comment.search(line):
- return True
- return _re_textual_blocks.search(line) is not None
+ Returns:
+ `str`: The styled docstring
+ """
+ lines = docstring.split("\n")
+ new_lines = []
- def is_special_block(self, line):
- """Whether or not `line` introduces a special block."""
- if self.is_no_style_block(line):
- self.in_block = SpecialBlock.NO_STYLE
- return True
- return False
+ # Initialization
+ current_paragraph = None
+ current_indent = -1
+ in_code = False
+ param_indent = -1
+ prefix = ""
- def init_in_block(self, text):
- """
- Returns the initial value for `self.in_block`.
-
- Useful for some docstrings beginning inside an argument declaration block (all models).
- """
- return SpecialBlock.NOT_SPECIAL
-
- def end_of_special_style(self, line):
- """
- Sets back the `in_block` attribute to `NOT_SPECIAL`.
-
- Useful for some docstrings where we may have to go back to `ARG_LIST` instead.
- """
- self.in_block = SpecialBlock.NOT_SPECIAL
-
- def style_paragraph(self, paragraph, max_len, no_style=False, min_indent=None):
- """
- Style `paragraph` (a list of lines) by making sure no line goes over `max_len`, except if the `no_style` flag
- is passed.
- """
- if len(paragraph) == 0:
- return ""
- if no_style or self.in_block == SpecialBlock.NO_STYLE:
- return "\n".join(paragraph)
- if _re_list.search(paragraph[0]) is not None:
- # Great, we're in a list. So we need to split our paragraphs in smaller parts, one for each item.
- result = ""
- remainder = ""
- prefix = _re_list.search(paragraph[0]).groups()[0]
- prefix_indent = get_indent(paragraph[0])
- current_item = [paragraph[0][len(prefix) :]]
- for i, line in enumerate(paragraph[1:]):
- new_item_search = _re_list.search(line)
- indent = get_indent(line)
- if len(indent) < len(prefix_indent) or (len(indent) == len(prefix_indent) and new_item_search is None):
- # There might not be an empty line after the list, formatting the remainder recursively.
- remainder = "\n" + self.style_paragraph(
- paragraph[i + 1 :], max_len, no_style=no_style, min_indent=min_indent
- )
- break
- elif new_item_search is not None:
- text = " ".join([l.strip() for l in current_item])
- result += split_text_in_lines(text, max_len, prefix, min_indent=min_indent) + "\n"
- prefix = new_item_search.groups()[0]
- prefix_indent = indent
- current_item = [line[len(prefix) :]]
- else:
- current_item.append(line)
- # Treat the last item
- text = " ".join([l.strip() for l in current_item])
- result += split_text_in_lines(text, max_len, prefix, min_indent=min_indent)
- # Add the potential remainder
- return result + remainder
-
- if len(paragraph) > 1 and self.is_comment_or_textual_block(paragraph[0]):
- # Comments/notes in rst should be restyled with indentation, ignoring the first line.
- indent = get_indent(paragraph[1])
- text = " ".join([l.strip() for l in paragraph[1:]])
- return paragraph[0] + "\n" + split_text_in_lines(text, max_len, indent, min_indent=min_indent)
-
- if self.in_block == SpecialBlock.ARG_LIST:
- # Arg lists are special: we need to ignore the lines that are at the first indentation level beneath the
- # Args/Parameters (parameter description), then we can style the indentation level beneath.
- result = ""
- # The args/parameters could be in that paragraph and should be ignored
- if _re_arg_def.search(paragraph[0]) is not None:
- if len(paragraph) == 1:
- return paragraph[0]
- result += paragraph[0] + "\n"
- paragraph = paragraph[1:]
-
- if self.current_indent is None:
- self.current_indent = get_indent(paragraph[1])
-
- current_item = []
- for line in paragraph:
- if get_indent(line) == self.current_indent:
- if len(current_item) > 0:
- item_indent = get_indent(current_item[0])
- text = " ".join([l.strip() for l in current_item])
- result += split_text_in_lines(text, max_len, item_indent, min_indent=min_indent) + "\n"
- result += line + "\n"
- current_item = []
- else:
- current_item.append(line)
- if len(current_item) > 0:
- item_indent = get_indent(current_item[0])
- text = " ".join([l.strip() for l in current_item])
- result += split_text_in_lines(text, max_len, item_indent, min_indent=min_indent) + "\n"
- return result[:-1]
-
- indent = get_indent(paragraph[0])
- text = " ".join([l.strip() for l in paragraph])
- return split_text_in_lines(text, max_len, indent, min_indent=min_indent)
-
- def style(self, text, max_len=119, min_indent=None):
- """Style `text` to `max_len`."""
- new_lines = []
- paragraph = []
- self.current_indent = ""
- self.previous_indent = None
- # If one of those is True, the paragraph should not be touched (code samples, lists...)
- no_style = False
- no_style_next = False
- self.in_block = self.init_in_block(text)
- # If this is True, we force-break a paragraph, even if there is no new empty line.
- break_paragraph = False
-
- lines = text.split("\n")
- last_line = None
- for line in lines:
- # New paragraph
- line_is_empty = len(line.strip()) == 0
- list_begins = (
- _re_list.search(line) is not None
- and last_line is not None
- and len(get_indent(line)) > len(get_indent(last_line))
- )
- if line_is_empty or break_paragraph or list_begins:
- if len(paragraph) > 0:
- if self.in_block != SpecialBlock.NOT_SPECIAL:
- indent = get_indent(paragraph[0])
- # Are we still in a no-style block?
- if self.current_indent is None:
- # If current_indent is None, we haven't begun the interior of the block so the answer is
- # yes, unless we have an indent of 0 in which case the special block took one line only.
- if len(indent) == 0:
- self.in_block = SpecialBlock.NOT_SPECIAL
- else:
- self.current_indent = indent
- elif not indent.startswith(self.current_indent):
- # If not, we are leaving the block when we unindent.
- self.end_of_special_style(paragraph[0])
-
- if self.is_special_block(paragraph[0]):
- # Maybe we are starting a special block.
- if len(paragraph) > 1:
- # If we have the interior of the block in the paragraph, we grab the indent.
- self.current_indent = get_indent(paragraph[1])
- else:
- # We will determine the indent with the next paragraph
- self.current_indent = None
- styled_paragraph = self.style_paragraph(
- paragraph, max_len, no_style=no_style, min_indent=min_indent
- )
- new_lines.append(styled_paragraph + "\n")
- else:
- new_lines.append("")
-
- paragraph = []
- no_style = no_style_next
- no_style_next = False
- last_line = None
- if (not break_paragraph and not list_begins) or line_is_empty:
- break_paragraph = False
- continue
- break_paragraph = False
-
- # Title and section lines should go to the max + add a new paragraph.
- if (
- len(set(line)) == 1
- and line[0] in TITLE_SPECIAL_CHARS
- and last_line is not None
- and len(line) >= len(last_line)
- ):
- line = line[0] * max_len
- break_paragraph = True
- # proper doc comment indicates the next paragraph should be no-style.
- if _re_doc_ignore.search(line) is not None:
- no_style_next = True
- # Table are in just one paragraph and should be no-style.
- if _re_table.search(line) is not None:
- no_style = True
- paragraph.append(line)
- last_line = line
-
- # Just have to treat the last paragraph. It could still be in a no-style block (or not)
- if len(paragraph) > 0:
- # Are we still in a special block
- # (if current_indent is None, we are but no need to set it since we are the end.)
- if self.in_block != SpecialBlock.NO_STYLE and self.current_indent is not None:
- indent = get_indent(paragraph[0])
- if not indent.startswith(self.current_indent):
- self.in_block = SpecialBlock.NOT_SPECIAL
- _ = self.is_special_block(paragraph[0])
- new_lines.append(self.style_paragraph(paragraph, max_len, no_style=no_style, min_indent=min_indent) + "\n")
- return "\n".join(new_lines)
-
-
-class DocstringStyler(CodeStyler):
- """Class to style docstrings that take the main method from `CodeStyler`."""
-
- def is_no_style_block(self, line):
- if _re_textual_blocks.search(line) is not None:
- return False
- if _re_example.search(line) is not None:
- return True
- return _re_code_block.search(line) is not None
-
- def is_comment_or_textual_block(self, line):
- if _re_return.search(line) is not None:
- self.in_block = SpecialBlock.NOT_SPECIAL
- return True
- return super().is_comment_or_textual_block(line)
-
- def is_special_block(self, line):
- if self.is_no_style_block(line):
- if self.previous_indent is None and self.in_block == SpecialBlock.ARG_LIST:
- self.previous_indent = self.current_indent
- self.in_block = SpecialBlock.NO_STYLE
- return True
- if _re_arg_def.search(line) is not None:
- self.in_block = SpecialBlock.ARG_LIST
- return True
- return False
-
- def end_of_special_style(self, line):
- if self.previous_indent is not None and line.startswith(self.previous_indent):
- self.in_block = SpecialBlock.ARG_LIST
- self.current_indent = self.previous_indent
- else:
- self.in_block = SpecialBlock.NOT_SPECIAL
- self.previous_indent = None
-
- def init_in_block(self, text):
- lines = text.split("\n")
- while len(lines) > 0 and len(lines[0]) == 0:
- lines = lines[1:]
- if len(lines) == 0:
- return SpecialBlock.NOT_SPECIAL
- if re.search(r":\s*$", lines[0]):
- indent = get_indent(lines[0])
- if (
- len(lines) == 1
- or len(get_indent(lines[1])) > len(indent)
- or (len(get_indent(lines[1])) == len(indent) and re.search(r":\s*$", lines[1]))
- ):
- self.current_indent = indent
- return SpecialBlock.ARG_LIST
- return SpecialBlock.NOT_SPECIAL
-
-
-rst_styler = CodeStyler()
-doc_styler = DocstringStyler()
-
-
-def _reindent_code_blocks(text):
- """Checks indent in code blocks is of four"""
- lines = text.split("\n")
+ # Special case for docstrings that begin with continuation of Args with no Args block.
idx = 0
- while idx < len(lines):
- # Detect if the line is the start of a new code-block.
- if _re_code_block.search(lines[idx]) is not None or _re_code_block_explicit.search(lines[idx]) is not None:
- while len(get_indent(lines[idx])) == 0:
- idx += 1
- indent = len(get_indent(lines[idx]))
- should_continue = True
- while should_continue:
- if len(lines[idx]) > 0 and indent < 4:
- lines[idx] = " " * 4 + lines[idx][indent:]
- idx += 1
- should_continue = (idx < len(lines)) and (len(lines[idx]) == 0 or len(get_indent(lines[idx])) > 0)
- else:
- idx += 1
+ while idx < len(lines) and is_empty_line(lines[idx]):
+ idx += 1
+ if (
+ len(lines[idx]) > 1
+ and lines[idx].rstrip().endswith(":")
+ and find_indent(lines[idx + 1]) > find_indent(lines[idx])
+ ):
+ param_indent = find_indent(lines[idx])
- return "\n".join(lines)
-
-
-def _add_new_lines_before_list(text):
- """Add a new empty line before a list begins."""
- lines = text.split("\n")
- new_lines = []
- in_list = False
for idx, line in enumerate(lines):
- # Detect if the line is the start of a new list.
- if _re_list.search(line) is not None and not in_list:
- current_indent = get_indent(line)
- in_list = True
- # If the line before is non empty, add an extra new line.
- if idx > 0 and len(lines[idx - 1]) != 0:
- new_lines.append("")
- # Detect if we're out of the current list.
- if in_list and not line.startswith(current_indent) and _re_list.search(line) is None:
- in_list = False
- new_lines.append(line)
+ # Doing all re searches once for the one we need to repeat.
+ list_search = _re_list.search(line)
+ code_search = _re_code.search(line)
+
+ # Are we starting a new paragraph?
+ # New indentation or new line:
+ new_paragraph = find_indent(line) != current_indent or is_empty_line(line)
+ # List item
+ new_paragraph = new_paragraph or list_search is not None
+ # Code block beginning
+ new_paragraph = new_paragraph or code_search is not None
+
+ # In this case, we treat the current paragraph
+ if not in_code and new_paragraph and current_paragraph is not None and len(current_paragraph) > 0:
+ paragraph = " ".join(current_paragraph)
+ new_lines.append(format_text(paragraph, max_len, prefix=prefix, min_indent=current_indent))
+ current_paragraph = None
+
+ if code_search is not None:
+ if not in_code:
+ current_paragraph = []
+ current_indent = len(code_search.groups()[0])
+ current_code = code_search.groups()[1]
+ prefix = ""
+ if current_indent < param_indent:
+ param_indent = -1
+ else:
+ current_indent = -1
+ code = "\n".join(current_paragraph)
+ if current_code in ["py", "python"]:
+ new_lines.append(code)
+ # new_lines.append(format_code_example(code, max_len))
+ else:
+ new_lines.append(code)
+ current_paragraph = None
+ new_lines.append(line)
+ in_code = not in_code
+
+ elif in_code:
+ current_paragraph.append(line)
+ elif is_empty_line(line):
+ current_paragraph = None
+ current_indent = -1
+ prefix = ""
+ new_lines.append(line)
+ elif list_search is not None:
+ prefix = list_search.groups()[0]
+ current_indent = len(prefix)
+ current_paragraph = [line[current_indent:]]
+ elif _re_args.search(line):
+ new_lines.append(line)
+ param_indent = find_indent(lines[idx + 1])
+ elif current_paragraph is None or find_indent(line) != current_indent:
+ indent = find_indent(line)
+ # Special behavior for parameters intros.
+ if indent == param_indent:
+ # Special rules for some docstring where the Returns blocks has the same indent as the parameters.
+ if _re_returns.search(line) is not None:
+ param_indent = -1
+ new_lines.append(line)
+ elif len(line) < max_len:
+ new_lines.append(line)
+ else:
+ intro, description = split_line_on_first_colon(line)
+ new_lines.append(intro + ":")
+ if len(description) != 0:
+ if find_indent(lines[idx + 1]) > indent:
+ current_indent = find_indent(lines[idx + 1])
+ else:
+ current_indent = indent + 4
+ current_paragraph = [description.strip()]
+ prefix = ""
+ else:
+ # Check if we have exited the parameter block
+ if indent < param_indent:
+ param_indent = -1
+
+ current_paragraph = [line.strip()]
+ current_indent = find_indent(line)
+ prefix = ""
+ elif current_paragraph is not None:
+ current_paragraph.append(line.lstrip())
+
+ if current_paragraph is not None and len(current_paragraph) > 0:
+ paragraph = " ".join(current_paragraph)
+ new_lines.append(format_text(paragraph, max_len, prefix=prefix, min_indent=current_indent))
+
return "\n".join(new_lines)
-def _add_new_lines_before_doc_special_words(text):
- lines = text.split("\n")
- new_lines = []
- for idx, line in enumerate(lines):
- # Detect if the line is the start of a new list.
- if _re_any_doc_special_word.search(line) is not None:
- # If the line before is non empty, add an extra new line.
- if idx > 0 and len(lines[idx - 1]) != 0:
- new_lines.append("")
- new_lines.append(line)
- return "\n".join(new_lines)
-
-
-def style_rst_file(doc_file, max_len=119, check_only=False):
- """Style one rst file `doc_file` to `max_len`."""
- with open(doc_file, "r", encoding="utf-8", newline="\n") as f:
- doc = f.read()
-
- # Make sure code blocks are indented at 4
- clean_doc = _reindent_code_blocks(doc)
- # Add missing new lines before lists
- clean_doc = _add_new_lines_before_list(clean_doc)
- # Style
- clean_doc = rst_styler.style(clean_doc, max_len=max_len)
-
- diff = clean_doc != doc
- if not check_only and diff:
- print(f"Overwriting content of {doc_file}.")
- with open(doc_file, "w", encoding="utf-8", newline="\n") as f:
- f.write(clean_doc)
-
- return diff
-
-
-def style_docstring(docstring, max_len=119):
- """Style `docstring` to `max_len`."""
- # One-line docstring that are not too long are left as is.
- if len(docstring) < max_len and "\n" not in docstring:
- return docstring
-
- # Grab the indent from the last line
- last_line = docstring.split("\n")[-1]
- # Is it empty except for the last triple-quotes (not-included in `docstring`)?
- indent_search = re.search(r"^(\s*)$", last_line)
- if indent_search is not None:
- indent = indent_search.groups()[0]
- if len(indent) > 0:
- docstring = docstring[: -len(indent)]
- # Or are the triple quotes next to text (we will fix that).
- else:
- indent_search = _re_indent.search(last_line)
- indent = indent_search.groups()[0] if indent_search is not None else ""
-
- # Add missing new lines before Args/Returns etc.
- docstring = _add_new_lines_before_doc_special_words(docstring)
- # Add missing new lines before lists
- docstring = _add_new_lines_before_list(docstring)
- # Style
- styled_doc = doc_styler.style(docstring, max_len=max_len, min_indent=indent)
-
- # Add new lines if necessary
- if not styled_doc.startswith("\n"):
- styled_doc = "\n" + styled_doc
- if not styled_doc.endswith("\n"):
- styled_doc += "\n"
- return styled_doc + indent
-
-
def style_file_docstrings(code_file, max_len=119, check_only=False):
- """Style all docstrings in `code_file` to `max_len`."""
+ """
+ Style all docstrings in a given file.
+
+ Args:
+ code_file (`str` or `os.PathLike`): The file in which we want to style the docstring.
+ max_len (`int`): The maximum number of characters per line.
+ check_only (`bool`, *optional*, defaults to `False`):
+ Whether to restyle file or just check if they should be restyled.
+
+ Returns:
+ `bool`: Whether or not the file was or should be restyled.
+ """
with open(code_file, "r", encoding="utf-8", newline="\n") as f:
code = f.read()
# fmt: off
@@ -507,10 +246,68 @@ def style_file_docstrings(code_file, max_len=119, check_only=False):
return diff
+def style_mdx_file(mdx_file, max_len=119, check_only=False):
+ """
+ Style a MDX file by formatting all Python code samples.
+
+ Args:
+ mdx_file (`str` or `os.PathLike`): The file in which we want to style the examples.
+ max_len (`int`): The maximum number of characters per line.
+ check_only (`bool`, *optional*, defaults to `False`):
+ Whether to restyle file or just check if they should be restyled.
+
+ Returns:
+ `bool`: Whether or not the file was or should be restyled.
+ """
+ with open(mdx_file, "r", encoding="utf-8", newline="\n") as f:
+ content = f.read()
+
+ lines = content.split("\n")
+ current_code = []
+ current_language = ""
+ in_code = False
+ new_lines = []
+ for line in lines:
+ if _re_code.search(line) is not None:
+ in_code = not in_code
+ if in_code:
+ current_language = _re_code.search(line).groups()[1]
+ current_code = []
+ else:
+ code = "\n".join(current_code)
+ if current_language in ["py", "python"]:
+ pass
+ # code = format_code_example(code, max_len)
+ new_lines.append(code)
+
+ new_lines.append(line)
+ elif in_code:
+ current_code.append(line)
+ else:
+ new_lines.append(line)
+
+ clean_content = "\n".join(new_lines)
+ diff = clean_content != content
+ if not check_only and diff:
+ print(f"Overwriting content of {mdx_file}.")
+ with open(mdx_file, "w", encoding="utf-8", newline="\n") as f:
+ f.write(clean_content)
+
+ return diff
+
+
def style_doc_files(*files, max_len=119, check_only=False):
"""
- Style all `files` to `max_len` and fixes mistakes if not `check_only`, otherwise raises an error if styling should
- be done.
+ Applies doc styling or checks everything is correct in a list of files.
+
+ Args:
+ files (several `str` or `os.PathLike`): The files to treat.
+ max_len (`int`): The maximum number of characters per line.
+ check_only (`bool`, *optional*, defaults to `False`):
+ Whether to restyle file or just check if they should be restyled.
+
+ Returns:
+ List[`str`]: The list of files changed or that should be restyled.
"""
changed = []
for file in files:
@@ -519,16 +316,20 @@ def style_doc_files(*files, max_len=119, check_only=False):
files = [os.path.join(file, f) for f in os.listdir(file)]
files = [f for f in files if os.path.isdir(f) or f.endswith(".rst") or f.endswith(".py")]
changed += style_doc_files(*files, max_len=max_len, check_only=check_only)
- # Treat rst
- elif file.endswith(".rst"):
- if style_rst_file(file, max_len=max_len, check_only=check_only):
+ # Treat mdx
+ elif file.endswith(".mdx"):
+ if style_mdx_file(file, max_len=max_len, check_only=check_only):
changed.append(file)
# Treat python files
elif file.endswith(".py"):
- if style_file_docstrings(file, max_len=max_len, check_only=check_only):
- changed.append(file)
+ try:
+ if style_file_docstrings(file, max_len=max_len, check_only=check_only):
+ changed.append(file)
+ except Exception:
+ print(f"There is a problem in {file}.")
+ raise
else:
- warnings.warn(f"Ignoring {file} because it's not a py or an rst file or a folder.")
+ warnings.warn(f"Ignoring {file} because it's not a py or an mdx file or a folder.")
return changed