From 3b1bbefc476cdd710273c79f55f521a6d91083c3 Mon Sep 17 00:00:00 2001 From: Yih-Dar <2521628+ydshieh@users.noreply.github.com> Date: Fri, 22 Apr 2022 10:53:24 +0200 Subject: [PATCH] Add missing entries in mappings (#16857) * add missing entries in some mappings Co-authored-by: ydshieh --- .../models/auto/feature_extraction_auto.py | 4 ++++ .../models/auto/processing_auto.py | 6 ++++++ .../models/auto/tokenization_auto.py | 18 ++++++++++++++++++ 3 files changed, 28 insertions(+) diff --git a/src/transformers/models/auto/feature_extraction_auto.py b/src/transformers/models/auto/feature_extraction_auto.py index dad7e165e8..552e3b7ae4 100644 --- a/src/transformers/models/auto/feature_extraction_auto.py +++ b/src/transformers/models/auto/feature_extraction_auto.py @@ -57,6 +57,10 @@ FEATURE_EXTRACTOR_MAPPING_NAMES = OrderedDict( ("regnet", "ConvNextFeatureExtractor"), ("poolformer", "PoolFormerFeatureExtractor"), ("maskformer", "MaskFormerFeatureExtractor"), + ("data2vec-audio", "Wav2Vec2FeatureExtractor"), + ("data2vec-vision", "BeitFeatureExtractor"), + ("dpt", "DPTFeatureExtractor"), + ("glpn", "GLPNFeatureExtractor"), ] ) diff --git a/src/transformers/models/auto/processing_auto.py b/src/transformers/models/auto/processing_auto.py index b51ef9ef31..ca49db286a 100644 --- a/src/transformers/models/auto/processing_auto.py +++ b/src/transformers/models/auto/processing_auto.py @@ -46,6 +46,12 @@ PROCESSOR_MAPPING_NAMES = OrderedDict( ("wav2vec2", "Wav2Vec2Processor"), ("wav2vec2_with_lm", "Wav2Vec2ProcessorWithLM"), ("vision-text-dual-encoder", "VisionTextDualEncoderProcessor"), + ("unispeech", "Wav2Vec2Processor"), + ("unispeech-sat", "Wav2Vec2Processor"), + ("sew", "Wav2Vec2Processor"), + ("sew-d", "Wav2Vec2Processor"), + ("vilt", "ViltProcessor"), + ("wavlm", "Wav2Vec2Processor"), ] ) diff --git a/src/transformers/models/auto/tokenization_auto.py b/src/transformers/models/auto/tokenization_auto.py index fb275a2037..0c0059c7c6 100644 --- a/src/transformers/models/auto/tokenization_auto.py +++ b/src/transformers/models/auto/tokenization_auto.py @@ -241,6 +241,24 @@ else: "XGLMTokenizerFast" if is_tokenizers_available() else None, ), ), + ("visual_bert", ("BertTokenizer", "BertTokenizerFast" if is_tokenizers_available() else None)), + ("megatron-bert", ("BertTokenizer", "BertTokenizerFast" if is_tokenizers_available() else None)), + ( + "nystromformer", + ( + "AlbertTokenizer" if is_sentencepiece_available() else None, + "AlbertTokenizerFast" if is_tokenizers_available() else None, + ), + ), + ("xlm-roberta-xl", ("RobertaTokenizer", "RobertaTokenizerFast" if is_tokenizers_available() else None)), + ( + "yoso", + ( + "AlbertTokenizer" if is_sentencepiece_available() else None, + "AlbertTokenizerFast" if is_tokenizers_available() else None, + ), + ), + ("data2vec-text", ("RobertaTokenizer", "RobertaTokenizerFast" if is_tokenizers_available() else None)), ] )