From a442f87adc3b60dbfea009eafa56bcabab364e1f Mon Sep 17 00:00:00 2001 From: Suraj Patil Date: Thu, 13 Aug 2020 21:36:43 +0530 Subject: [PATCH] add LongformerTokenizerFast in AutoTokenizer (#6463) --- src/transformers/tokenization_auto.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/transformers/tokenization_auto.py b/src/transformers/tokenization_auto.py index dd35746c6a..6b6c64d9fe 100644 --- a/src/transformers/tokenization_auto.py +++ b/src/transformers/tokenization_auto.py @@ -55,7 +55,7 @@ from .tokenization_distilbert import DistilBertTokenizer, DistilBertTokenizerFas from .tokenization_electra import ElectraTokenizer, ElectraTokenizerFast from .tokenization_flaubert import FlaubertTokenizer from .tokenization_gpt2 import GPT2Tokenizer, GPT2TokenizerFast -from .tokenization_longformer import LongformerTokenizer +from .tokenization_longformer import LongformerTokenizer, LongformerTokenizerFast from .tokenization_marian import MarianTokenizer from .tokenization_mobilebert import MobileBertTokenizer, MobileBertTokenizerFast from .tokenization_openai import OpenAIGPTTokenizer, OpenAIGPTTokenizerFast @@ -86,7 +86,7 @@ TOKENIZER_MAPPING = OrderedDict( (XLMRobertaConfig, (XLMRobertaTokenizer, None)), (MarianConfig, (MarianTokenizer, None)), (BartConfig, (BartTokenizer, None)), - (LongformerConfig, (LongformerTokenizer, None)), + (LongformerConfig, (LongformerTokenizer, LongformerTokenizerFast)), (RobertaConfig, (RobertaTokenizer, RobertaTokenizerFast)), (ReformerConfig, (ReformerTokenizer, None)), (ElectraConfig, (ElectraTokenizer, ElectraTokenizerFast)),