From 640e1b6c6f78ed18ca8737d87c33ac6e2cc718a3 Mon Sep 17 00:00:00 2001 From: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> Date: Fri, 21 Jul 2023 09:41:36 -0400 Subject: [PATCH] Remove tokenizers from the doc table (#24963) --- docs/source/en/index.md | 400 ++++++++++++++++++++-------------------- utils/check_table.py | 14 +- 2 files changed, 202 insertions(+), 212 deletions(-) diff --git a/docs/source/en/index.md b/docs/source/en/index.md index ced49d3ed7..28139d406c 100644 --- a/docs/source/en/index.md +++ b/docs/source/en/index.md @@ -278,205 +278,205 @@ Flax), PyTorch, and/or TensorFlow. -| Model | Tokenizer slow | Tokenizer fast | PyTorch support | TensorFlow support | Flax Support | -|:-----------------------------:|:--------------:|:--------------:|:---------------:|:------------------:|:------------:| -| ALBERT | ✅ | ✅ | ✅ | ✅ | ✅ | -| ALIGN | ❌ | ❌ | ✅ | ❌ | ❌ | -| AltCLIP | ❌ | ❌ | ✅ | ❌ | ❌ | -| Audio Spectrogram Transformer | ❌ | ❌ | ✅ | ❌ | ❌ | -| Autoformer | ❌ | ❌ | ✅ | ❌ | ❌ | -| Bark | ❌ | ❌ | ✅ | ❌ | ❌ | -| BART | ✅ | ✅ | ✅ | ✅ | ✅ | -| BEiT | ❌ | ❌ | ✅ | ❌ | ✅ | -| BERT | ✅ | ✅ | ✅ | ✅ | ✅ | -| Bert Generation | ✅ | ❌ | ✅ | ❌ | ❌ | -| BigBird | ✅ | ✅ | ✅ | ❌ | ✅ | -| BigBird-Pegasus | ❌ | ❌ | ✅ | ❌ | ❌ | -| BioGpt | ✅ | ❌ | ✅ | ❌ | ❌ | -| BiT | ❌ | ❌ | ✅ | ❌ | ❌ | -| Blenderbot | ✅ | ✅ | ✅ | ✅ | ✅ | -| BlenderbotSmall | ✅ | ✅ | ✅ | ✅ | ✅ | -| BLIP | ❌ | ❌ | ✅ | ✅ | ❌ | -| BLIP-2 | ❌ | ❌ | ✅ | ❌ | ❌ | -| BLOOM | ❌ | ✅ | ✅ | ❌ | ❌ | -| BridgeTower | ❌ | ❌ | ✅ | ❌ | ❌ | -| CamemBERT | ✅ | ✅ | ✅ | ✅ | ❌ | -| CANINE | ✅ | ❌ | ✅ | ❌ | ❌ | -| Chinese-CLIP | ❌ | ❌ | ✅ | ❌ | ❌ | -| CLAP | ❌ | ❌ | ✅ | ❌ | ❌ | -| CLIP | ✅ | ✅ | ✅ | ✅ | ✅ | -| CLIPSeg | ❌ | ❌ | ✅ | ❌ | ❌ | -| CodeGen | ✅ | ✅ | ✅ | ❌ | ❌ | -| Conditional DETR | ❌ | ❌ | ✅ | ❌ | ❌ | -| ConvBERT | ✅ | ✅ | ✅ | ✅ | ❌ | -| ConvNeXT | ❌ | ❌ | ✅ | ✅ | ❌ | -| ConvNeXTV2 | ❌ | ❌ | ✅ | ❌ | ❌ | -| CPM-Ant | ✅ | ❌ | ✅ | ❌ | ❌ | -| CTRL | ✅ | ❌ | ✅ | ✅ | ❌ | -| CvT | ❌ | ❌ | ✅ | ✅ | ❌ | -| Data2VecAudio | ❌ | ❌ | ✅ | ❌ | ❌ | -| Data2VecText | ❌ | ❌ | ✅ | ❌ | ❌ | -| Data2VecVision | ❌ | ❌ | ✅ | ✅ | ❌ | -| DeBERTa | ✅ | ✅ | ✅ | ✅ | ❌ | -| DeBERTa-v2 | ✅ | ✅ | ✅ | ✅ | ❌ | -| Decision Transformer | ❌ | ❌ | ✅ | ❌ | ❌ | -| Deformable DETR | ❌ | ❌ | ✅ | ❌ | ❌ | -| DeiT | ❌ | ❌ | ✅ | ✅ | ❌ | -| DETA | ❌ | ❌ | ✅ | ❌ | ❌ | -| DETR | ❌ | ❌ | ✅ | ❌ | ❌ | -| DiNAT | ❌ | ❌ | ✅ | ❌ | ❌ | -| DINOv2 | ❌ | ❌ | ✅ | ❌ | ❌ | -| DistilBERT | ✅ | ✅ | ✅ | ✅ | ✅ | -| DonutSwin | ❌ | ❌ | ✅ | ❌ | ❌ | -| DPR | ✅ | ✅ | ✅ | ✅ | ❌ | -| DPT | ❌ | ❌ | ✅ | ❌ | ❌ | -| EfficientFormer | ❌ | ❌ | ✅ | ✅ | ❌ | -| EfficientNet | ❌ | ❌ | ✅ | ❌ | ❌ | -| ELECTRA | ✅ | ✅ | ✅ | ✅ | ✅ | -| EnCodec | ❌ | ❌ | ✅ | ❌ | ❌ | -| Encoder decoder | ❌ | ❌ | ✅ | ✅ | ✅ | -| ERNIE | ❌ | ❌ | ✅ | ❌ | ❌ | -| ErnieM | ✅ | ❌ | ✅ | ❌ | ❌ | -| ESM | ✅ | ❌ | ✅ | ✅ | ❌ | -| FairSeq Machine-Translation | ✅ | ❌ | ✅ | ❌ | ❌ | -| Falcon | ❌ | ❌ | ✅ | ❌ | ❌ | -| FlauBERT | ✅ | ❌ | ✅ | ✅ | ❌ | -| FLAVA | ❌ | ❌ | ✅ | ❌ | ❌ | -| FNet | ✅ | ✅ | ✅ | ❌ | ❌ | -| FocalNet | ❌ | ❌ | ✅ | ❌ | ❌ | -| Funnel Transformer | ✅ | ✅ | ✅ | ✅ | ❌ | -| GIT | ❌ | ❌ | ✅ | ❌ | ❌ | -| GLPN | ❌ | ❌ | ✅ | ❌ | ❌ | -| GPT Neo | ❌ | ❌ | ✅ | ❌ | ✅ | -| GPT NeoX | ❌ | ✅ | ✅ | ❌ | ❌ | -| GPT NeoX Japanese | ✅ | ❌ | ✅ | ❌ | ❌ | -| GPT-J | ❌ | ❌ | ✅ | ✅ | ✅ | -| GPT-Sw3 | ✅ | ✅ | ✅ | ✅ | ✅ | -| GPTBigCode | ❌ | ❌ | ✅ | ❌ | ❌ | -| GPTSAN-japanese | ✅ | ❌ | ✅ | ❌ | ❌ | -| Graphormer | ❌ | ❌ | ✅ | ❌ | ❌ | -| GroupViT | ❌ | ❌ | ✅ | ✅ | ❌ | -| Hubert | ❌ | ❌ | ✅ | ✅ | ❌ | -| I-BERT | ❌ | ❌ | ✅ | ❌ | ❌ | -| ImageGPT | ❌ | ❌ | ✅ | ❌ | ❌ | -| Informer | ❌ | ❌ | ✅ | ❌ | ❌ | -| InstructBLIP | ❌ | ❌ | ✅ | ❌ | ❌ | -| Jukebox | ✅ | ❌ | ✅ | ❌ | ❌ | -| LayoutLM | ✅ | ✅ | ✅ | ✅ | ❌ | -| LayoutLMv2 | ✅ | ✅ | ✅ | ❌ | ❌ | -| LayoutLMv3 | ✅ | ✅ | ✅ | ✅ | ❌ | -| LED | ✅ | ✅ | ✅ | ✅ | ❌ | -| LeViT | ❌ | ❌ | ✅ | ❌ | ❌ | -| LiLT | ❌ | ❌ | ✅ | ❌ | ❌ | -| LLaMA | ✅ | ✅ | ✅ | ❌ | ❌ | -| Longformer | ✅ | ✅ | ✅ | ✅ | ❌ | -| LongT5 | ❌ | ❌ | ✅ | ❌ | ✅ | -| LUKE | ✅ | ❌ | ✅ | ❌ | ❌ | -| LXMERT | ✅ | ✅ | ✅ | ✅ | ❌ | -| M-CTC-T | ❌ | ❌ | ✅ | ❌ | ❌ | -| M2M100 | ✅ | ❌ | ✅ | ❌ | ❌ | -| Marian | ✅ | ❌ | ✅ | ✅ | ✅ | -| MarkupLM | ✅ | ✅ | ✅ | ❌ | ❌ | -| Mask2Former | ❌ | ❌ | ✅ | ❌ | ❌ | -| MaskFormer | ❌ | ❌ | ✅ | ❌ | ❌ | -| MaskFormerSwin | ❌ | ❌ | ❌ | ❌ | ❌ | -| mBART | ✅ | ✅ | ✅ | ✅ | ✅ | -| MEGA | ❌ | ❌ | ✅ | ❌ | ❌ | -| Megatron-BERT | ❌ | ❌ | ✅ | ❌ | ❌ | -| MGP-STR | ✅ | ❌ | ✅ | ❌ | ❌ | -| MobileBERT | ✅ | ✅ | ✅ | ✅ | ❌ | -| MobileNetV1 | ❌ | ❌ | ✅ | ❌ | ❌ | -| MobileNetV2 | ❌ | ❌ | ✅ | ❌ | ❌ | -| MobileViT | ❌ | ❌ | ✅ | ✅ | ❌ | -| MobileViTV2 | ❌ | ❌ | ✅ | ❌ | ❌ | -| MPNet | ✅ | ✅ | ✅ | ✅ | ❌ | -| MRA | ❌ | ❌ | ✅ | ❌ | ❌ | -| MT5 | ✅ | ✅ | ✅ | ✅ | ✅ | -| MusicGen | ❌ | ❌ | ✅ | ❌ | ❌ | -| MVP | ✅ | ✅ | ✅ | ❌ | ❌ | -| NAT | ❌ | ❌ | ✅ | ❌ | ❌ | -| Nezha | ❌ | ❌ | ✅ | ❌ | ❌ | -| NLLB-MOE | ❌ | ❌ | ✅ | ❌ | ❌ | -| Nyströmformer | ❌ | ❌ | ✅ | ❌ | ❌ | -| OneFormer | ❌ | ❌ | ✅ | ❌ | ❌ | -| OpenAI GPT | ✅ | ✅ | ✅ | ✅ | ❌ | -| OpenAI GPT-2 | ✅ | ✅ | ✅ | ✅ | ✅ | -| OpenLlama | ❌ | ❌ | ✅ | ❌ | ❌ | -| OPT | ❌ | ❌ | ✅ | ✅ | ✅ | -| OWL-ViT | ❌ | ❌ | ✅ | ❌ | ❌ | -| Pegasus | ✅ | ✅ | ✅ | ✅ | ✅ | -| PEGASUS-X | ❌ | ❌ | ✅ | ❌ | ❌ | -| Perceiver | ✅ | ❌ | ✅ | ❌ | ❌ | -| Pix2Struct | ❌ | ❌ | ✅ | ❌ | ❌ | -| PLBart | ✅ | ❌ | ✅ | ❌ | ❌ | -| PoolFormer | ❌ | ❌ | ✅ | ❌ | ❌ | -| ProphetNet | ✅ | ❌ | ✅ | ❌ | ❌ | -| QDQBert | ❌ | ❌ | ✅ | ❌ | ❌ | -| RAG | ✅ | ❌ | ✅ | ✅ | ❌ | -| REALM | ✅ | ✅ | ✅ | ❌ | ❌ | -| Reformer | ✅ | ✅ | ✅ | ❌ | ❌ | -| RegNet | ❌ | ❌ | ✅ | ✅ | ✅ | -| RemBERT | ✅ | ✅ | ✅ | ✅ | ❌ | -| ResNet | ❌ | ❌ | ✅ | ✅ | ✅ | -| RetriBERT | ✅ | ✅ | ✅ | ❌ | ❌ | -| RoBERTa | ✅ | ✅ | ✅ | ✅ | ✅ | -| RoBERTa-PreLayerNorm | ❌ | ❌ | ✅ | ✅ | ✅ | -| RoCBert | ✅ | ❌ | ✅ | ❌ | ❌ | -| RoFormer | ✅ | ✅ | ✅ | ✅ | ✅ | -| RWKV | ❌ | ❌ | ✅ | ❌ | ❌ | -| SAM | ❌ | ❌ | ✅ | ✅ | ❌ | -| SegFormer | ❌ | ❌ | ✅ | ✅ | ❌ | -| SEW | ❌ | ❌ | ✅ | ❌ | ❌ | -| SEW-D | ❌ | ❌ | ✅ | ❌ | ❌ | -| Speech Encoder decoder | ❌ | ❌ | ✅ | ❌ | ✅ | -| Speech2Text | ✅ | ❌ | ✅ | ✅ | ❌ | -| Speech2Text2 | ✅ | ❌ | ❌ | ❌ | ❌ | -| SpeechT5 | ✅ | ❌ | ✅ | ❌ | ❌ | -| Splinter | ✅ | ✅ | ✅ | ❌ | ❌ | -| SqueezeBERT | ✅ | ✅ | ✅ | ❌ | ❌ | -| SwiftFormer | ❌ | ❌ | ✅ | ❌ | ❌ | -| Swin Transformer | ❌ | ❌ | ✅ | ✅ | ❌ | -| Swin Transformer V2 | ❌ | ❌ | ✅ | ❌ | ❌ | -| Swin2SR | ❌ | ❌ | ✅ | ❌ | ❌ | -| SwitchTransformers | ❌ | ❌ | ✅ | ❌ | ❌ | -| T5 | ✅ | ✅ | ✅ | ✅ | ✅ | -| Table Transformer | ❌ | ❌ | ✅ | ❌ | ❌ | -| TAPAS | ✅ | ❌ | ✅ | ✅ | ❌ | -| Time Series Transformer | ❌ | ❌ | ✅ | ❌ | ❌ | -| TimeSformer | ❌ | ❌ | ✅ | ❌ | ❌ | -| TimmBackbone | ❌ | ❌ | ❌ | ❌ | ❌ | -| Trajectory Transformer | ❌ | ❌ | ✅ | ❌ | ❌ | -| Transformer-XL | ✅ | ❌ | ✅ | ✅ | ❌ | -| TrOCR | ❌ | ❌ | ✅ | ❌ | ❌ | -| TVLT | ❌ | ❌ | ✅ | ❌ | ❌ | -| UMT5 | ❌ | ❌ | ✅ | ❌ | ❌ | -| UniSpeech | ❌ | ❌ | ✅ | ❌ | ❌ | -| UniSpeechSat | ❌ | ❌ | ✅ | ❌ | ❌ | -| UPerNet | ❌ | ❌ | ✅ | ❌ | ❌ | -| VAN | ❌ | ❌ | ✅ | ❌ | ❌ | -| VideoMAE | ❌ | ❌ | ✅ | ❌ | ❌ | -| ViLT | ❌ | ❌ | ✅ | ❌ | ❌ | -| Vision Encoder decoder | ❌ | ❌ | ✅ | ✅ | ✅ | -| VisionTextDualEncoder | ❌ | ❌ | ✅ | ✅ | ✅ | -| VisualBERT | ❌ | ❌ | ✅ | ❌ | ❌ | -| ViT | ❌ | ❌ | ✅ | ✅ | ✅ | -| ViT Hybrid | ❌ | ❌ | ✅ | ❌ | ❌ | -| ViTMAE | ❌ | ❌ | ✅ | ✅ | ❌ | -| ViTMSN | ❌ | ❌ | ✅ | ❌ | ❌ | -| ViViT | ❌ | ❌ | ✅ | ❌ | ❌ | -| Wav2Vec2 | ✅ | ❌ | ✅ | ✅ | ✅ | -| Wav2Vec2-Conformer | ❌ | ❌ | ✅ | ❌ | ❌ | -| WavLM | ❌ | ❌ | ✅ | ❌ | ❌ | -| Whisper | ✅ | ✅ | ✅ | ✅ | ✅ | -| X-CLIP | ❌ | ❌ | ✅ | ❌ | ❌ | -| X-MOD | ❌ | ❌ | ✅ | ❌ | ❌ | -| XGLM | ✅ | ✅ | ✅ | ✅ | ✅ | -| XLM | ✅ | ❌ | ✅ | ✅ | ❌ | -| XLM-ProphetNet | ✅ | ❌ | ✅ | ❌ | ❌ | -| XLM-RoBERTa | ✅ | ✅ | ✅ | ✅ | ✅ | -| XLM-RoBERTa-XL | ❌ | ❌ | ✅ | ❌ | ❌ | -| XLNet | ✅ | ✅ | ✅ | ✅ | ❌ | -| YOLOS | ❌ | ❌ | ✅ | ❌ | ❌ | -| YOSO | ❌ | ❌ | ✅ | ❌ | ❌ | +| Model | PyTorch support | TensorFlow support | Flax Support | +|:-----------------------------:|:---------------:|:------------------:|:------------:| +| ALBERT | ✅ | ✅ | ✅ | +| ALIGN | ✅ | ❌ | ❌ | +| AltCLIP | ✅ | ❌ | ❌ | +| Audio Spectrogram Transformer | ✅ | ❌ | ❌ | +| Autoformer | ✅ | ❌ | ❌ | +| Bark | ✅ | ❌ | ❌ | +| BART | ✅ | ✅ | ✅ | +| BEiT | ✅ | ❌ | ✅ | +| BERT | ✅ | ✅ | ✅ | +| Bert Generation | ✅ | ❌ | ❌ | +| BigBird | ✅ | ❌ | ✅ | +| BigBird-Pegasus | ✅ | ❌ | ❌ | +| BioGpt | ✅ | ❌ | ❌ | +| BiT | ✅ | ❌ | ❌ | +| Blenderbot | ✅ | ✅ | ✅ | +| BlenderbotSmall | ✅ | ✅ | ✅ | +| BLIP | ✅ | ✅ | ❌ | +| BLIP-2 | ✅ | ❌ | ❌ | +| BLOOM | ✅ | ❌ | ❌ | +| BridgeTower | ✅ | ❌ | ❌ | +| CamemBERT | ✅ | ✅ | ❌ | +| CANINE | ✅ | ❌ | ❌ | +| Chinese-CLIP | ✅ | ❌ | ❌ | +| CLAP | ✅ | ❌ | ❌ | +| CLIP | ✅ | ✅ | ✅ | +| CLIPSeg | ✅ | ❌ | ❌ | +| CodeGen | ✅ | ❌ | ❌ | +| Conditional DETR | ✅ | ❌ | ❌ | +| ConvBERT | ✅ | ✅ | ❌ | +| ConvNeXT | ✅ | ✅ | ❌ | +| ConvNeXTV2 | ✅ | ❌ | ❌ | +| CPM-Ant | ✅ | ❌ | ❌ | +| CTRL | ✅ | ✅ | ❌ | +| CvT | ✅ | ✅ | ❌ | +| Data2VecAudio | ✅ | ❌ | ❌ | +| Data2VecText | ✅ | ❌ | ❌ | +| Data2VecVision | ✅ | ✅ | ❌ | +| DeBERTa | ✅ | ✅ | ❌ | +| DeBERTa-v2 | ✅ | ✅ | ❌ | +| Decision Transformer | ✅ | ❌ | ❌ | +| Deformable DETR | ✅ | ❌ | ❌ | +| DeiT | ✅ | ✅ | ❌ | +| DETA | ✅ | ❌ | ❌ | +| DETR | ✅ | ❌ | ❌ | +| DiNAT | ✅ | ❌ | ❌ | +| DINOv2 | ✅ | ❌ | ❌ | +| DistilBERT | ✅ | ✅ | ✅ | +| DonutSwin | ✅ | ❌ | ❌ | +| DPR | ✅ | ✅ | ❌ | +| DPT | ✅ | ❌ | ❌ | +| EfficientFormer | ✅ | ✅ | ❌ | +| EfficientNet | ✅ | ❌ | ❌ | +| ELECTRA | ✅ | ✅ | ✅ | +| EnCodec | ✅ | ❌ | ❌ | +| Encoder decoder | ✅ | ✅ | ✅ | +| ERNIE | ✅ | ❌ | ❌ | +| ErnieM | ✅ | ❌ | ❌ | +| ESM | ✅ | ✅ | ❌ | +| FairSeq Machine-Translation | ✅ | ❌ | ❌ | +| Falcon | ✅ | ❌ | ❌ | +| FlauBERT | ✅ | ✅ | ❌ | +| FLAVA | ✅ | ❌ | ❌ | +| FNet | ✅ | ❌ | ❌ | +| FocalNet | ✅ | ❌ | ❌ | +| Funnel Transformer | ✅ | ✅ | ❌ | +| GIT | ✅ | ❌ | ❌ | +| GLPN | ✅ | ❌ | ❌ | +| GPT Neo | ✅ | ❌ | ✅ | +| GPT NeoX | ✅ | ❌ | ❌ | +| GPT NeoX Japanese | ✅ | ❌ | ❌ | +| GPT-J | ✅ | ✅ | ✅ | +| GPT-Sw3 | ✅ | ✅ | ✅ | +| GPTBigCode | ✅ | ❌ | ❌ | +| GPTSAN-japanese | ✅ | ❌ | ❌ | +| Graphormer | ✅ | ❌ | ❌ | +| GroupViT | ✅ | ✅ | ❌ | +| Hubert | ✅ | ✅ | ❌ | +| I-BERT | ✅ | ❌ | ❌ | +| ImageGPT | ✅ | ❌ | ❌ | +| Informer | ✅ | ❌ | ❌ | +| InstructBLIP | ✅ | ❌ | ❌ | +| Jukebox | ✅ | ❌ | ❌ | +| LayoutLM | ✅ | ✅ | ❌ | +| LayoutLMv2 | ✅ | ❌ | ❌ | +| LayoutLMv3 | ✅ | ✅ | ❌ | +| LED | ✅ | ✅ | ❌ | +| LeViT | ✅ | ❌ | ❌ | +| LiLT | ✅ | ❌ | ❌ | +| LLaMA | ✅ | ❌ | ❌ | +| Longformer | ✅ | ✅ | ❌ | +| LongT5 | ✅ | ❌ | ✅ | +| LUKE | ✅ | ❌ | ❌ | +| LXMERT | ✅ | ✅ | ❌ | +| M-CTC-T | ✅ | ❌ | ❌ | +| M2M100 | ✅ | ❌ | ❌ | +| Marian | ✅ | ✅ | ✅ | +| MarkupLM | ✅ | ❌ | ❌ | +| Mask2Former | ✅ | ❌ | ❌ | +| MaskFormer | ✅ | ❌ | ❌ | +| MaskFormerSwin | ❌ | ❌ | ❌ | +| mBART | ✅ | ✅ | ✅ | +| MEGA | ✅ | ❌ | ❌ | +| Megatron-BERT | ✅ | ❌ | ❌ | +| MGP-STR | ✅ | ❌ | ❌ | +| MobileBERT | ✅ | ✅ | ❌ | +| MobileNetV1 | ✅ | ❌ | ❌ | +| MobileNetV2 | ✅ | ❌ | ❌ | +| MobileViT | ✅ | ✅ | ❌ | +| MobileViTV2 | ✅ | ❌ | ❌ | +| MPNet | ✅ | ✅ | ❌ | +| MRA | ✅ | ❌ | ❌ | +| MT5 | ✅ | ✅ | ✅ | +| MusicGen | ✅ | ❌ | ❌ | +| MVP | ✅ | ❌ | ❌ | +| NAT | ✅ | ❌ | ❌ | +| Nezha | ✅ | ❌ | ❌ | +| NLLB-MOE | ✅ | ❌ | ❌ | +| Nyströmformer | ✅ | ❌ | ❌ | +| OneFormer | ✅ | ❌ | ❌ | +| OpenAI GPT | ✅ | ✅ | ❌ | +| OpenAI GPT-2 | ✅ | ✅ | ✅ | +| OpenLlama | ✅ | ❌ | ❌ | +| OPT | ✅ | ✅ | ✅ | +| OWL-ViT | ✅ | ❌ | ❌ | +| Pegasus | ✅ | ✅ | ✅ | +| PEGASUS-X | ✅ | ❌ | ❌ | +| Perceiver | ✅ | ❌ | ❌ | +| Pix2Struct | ✅ | ❌ | ❌ | +| PLBart | ✅ | ❌ | ❌ | +| PoolFormer | ✅ | ❌ | ❌ | +| ProphetNet | ✅ | ❌ | ❌ | +| QDQBert | ✅ | ❌ | ❌ | +| RAG | ✅ | ✅ | ❌ | +| REALM | ✅ | ❌ | ❌ | +| Reformer | ✅ | ❌ | ❌ | +| RegNet | ✅ | ✅ | ✅ | +| RemBERT | ✅ | ✅ | ❌ | +| ResNet | ✅ | ✅ | ✅ | +| RetriBERT | ✅ | ❌ | ❌ | +| RoBERTa | ✅ | ✅ | ✅ | +| RoBERTa-PreLayerNorm | ✅ | ✅ | ✅ | +| RoCBert | ✅ | ❌ | ❌ | +| RoFormer | ✅ | ✅ | ✅ | +| RWKV | ✅ | ❌ | ❌ | +| SAM | ✅ | ✅ | ❌ | +| SegFormer | ✅ | ✅ | ❌ | +| SEW | ✅ | ❌ | ❌ | +| SEW-D | ✅ | ❌ | ❌ | +| Speech Encoder decoder | ✅ | ❌ | ✅ | +| Speech2Text | ✅ | ✅ | ❌ | +| Speech2Text2 | ❌ | ❌ | ❌ | +| SpeechT5 | ✅ | ❌ | ❌ | +| Splinter | ✅ | ❌ | ❌ | +| SqueezeBERT | ✅ | ❌ | ❌ | +| SwiftFormer | ✅ | ❌ | ❌ | +| Swin Transformer | ✅ | ✅ | ❌ | +| Swin Transformer V2 | ✅ | ❌ | ❌ | +| Swin2SR | ✅ | ❌ | ❌ | +| SwitchTransformers | ✅ | ❌ | ❌ | +| T5 | ✅ | ✅ | ✅ | +| Table Transformer | ✅ | ❌ | ❌ | +| TAPAS | ✅ | ✅ | ❌ | +| Time Series Transformer | ✅ | ❌ | ❌ | +| TimeSformer | ✅ | ❌ | ❌ | +| TimmBackbone | ❌ | ❌ | ❌ | +| Trajectory Transformer | ✅ | ❌ | ❌ | +| Transformer-XL | ✅ | ✅ | ❌ | +| TrOCR | ✅ | ❌ | ❌ | +| TVLT | ✅ | ❌ | ❌ | +| UMT5 | ✅ | ❌ | ❌ | +| UniSpeech | ✅ | ❌ | ❌ | +| UniSpeechSat | ✅ | ❌ | ❌ | +| UPerNet | ✅ | ❌ | ❌ | +| VAN | ✅ | ❌ | ❌ | +| VideoMAE | ✅ | ❌ | ❌ | +| ViLT | ✅ | ❌ | ❌ | +| Vision Encoder decoder | ✅ | ✅ | ✅ | +| VisionTextDualEncoder | ✅ | ✅ | ✅ | +| VisualBERT | ✅ | ❌ | ❌ | +| ViT | ✅ | ✅ | ✅ | +| ViT Hybrid | ✅ | ❌ | ❌ | +| ViTMAE | ✅ | ✅ | ❌ | +| ViTMSN | ✅ | ❌ | ❌ | +| ViViT | ✅ | ❌ | ❌ | +| Wav2Vec2 | ✅ | ✅ | ✅ | +| Wav2Vec2-Conformer | ✅ | ❌ | ❌ | +| WavLM | ✅ | ❌ | ❌ | +| Whisper | ✅ | ✅ | ✅ | +| X-CLIP | ✅ | ❌ | ❌ | +| X-MOD | ✅ | ❌ | ❌ | +| XGLM | ✅ | ✅ | ✅ | +| XLM | ✅ | ✅ | ❌ | +| XLM-ProphetNet | ✅ | ❌ | ❌ | +| XLM-RoBERTa | ✅ | ✅ | ✅ | +| XLM-RoBERTa-XL | ✅ | ❌ | ❌ | +| XLNet | ✅ | ✅ | ❌ | +| YOLOS | ✅ | ❌ | ❌ | +| YOSO | ✅ | ❌ | ❌ | diff --git a/utils/check_table.py b/utils/check_table.py index 77eabe4093..a48310f7c1 100644 --- a/utils/check_table.py +++ b/utils/check_table.py @@ -93,8 +93,6 @@ def get_model_table_from_auto_modules(): model_name_to_prefix = {name: config.replace("Config", "") for name, config in model_name_to_config.items()} # Dictionaries flagging if each model prefix has a slow/fast tokenizer, backend in PT/TF/Flax. - slow_tokenizers = collections.defaultdict(bool) - fast_tokenizers = collections.defaultdict(bool) pt_models = collections.defaultdict(bool) tf_models = collections.defaultdict(bool) flax_models = collections.defaultdict(bool) @@ -102,13 +100,7 @@ def get_model_table_from_auto_modules(): # Let's lookup through all transformers object (once). for attr_name in dir(transformers_module): lookup_dict = None - if attr_name.endswith("Tokenizer"): - lookup_dict = slow_tokenizers - attr_name = attr_name[:-9] - elif attr_name.endswith("TokenizerFast"): - lookup_dict = fast_tokenizers - attr_name = attr_name[:-13] - elif _re_tf_models.match(attr_name) is not None: + if _re_tf_models.match(attr_name) is not None: lookup_dict = tf_models attr_name = _re_tf_models.match(attr_name).groups()[0] elif _re_flax_models.match(attr_name) is not None: @@ -129,7 +121,7 @@ def get_model_table_from_auto_modules(): # Let's build that table! model_names = list(model_name_to_config.keys()) model_names.sort(key=str.lower) - columns = ["Model", "Tokenizer slow", "Tokenizer fast", "PyTorch support", "TensorFlow support", "Flax Support"] + columns = ["Model", "PyTorch support", "TensorFlow support", "Flax Support"] # We'll need widths to properly display everything in the center (+2 is to leave one extra space on each side). widths = [len(c) + 2 for c in columns] widths[0] = max([len(name) for name in model_names]) + 2 @@ -144,8 +136,6 @@ def get_model_table_from_auto_modules(): prefix = model_name_to_prefix[name] line = [ name, - check[slow_tokenizers[prefix]], - check[fast_tokenizers[prefix]], check[pt_models[prefix]], check[tf_models[prefix]], check[flax_models[prefix]],