From 245cdb469d2a7f47316926fdbac925e0ed149332 Mon Sep 17 00:00:00 2001 From: Lysandre Debut Date: Wed, 13 Jan 2021 06:24:10 -0500 Subject: [PATCH] Fix barthez tokenizer (#9562) --- src/transformers/models/auto/tokenization_auto.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/auto/tokenization_auto.py b/src/transformers/models/auto/tokenization_auto.py index f46380ca0a..31fd06c591 100644 --- a/src/transformers/models/auto/tokenization_auto.py +++ b/src/transformers/models/auto/tokenization_auto.py @@ -199,7 +199,6 @@ TOKENIZER_MAPPING = OrderedDict( (AlbertConfig, (AlbertTokenizer, AlbertTokenizerFast)), (CamembertConfig, (CamembertTokenizer, CamembertTokenizerFast)), (PegasusConfig, (PegasusTokenizer, PegasusTokenizerFast)), - (MBartConfig, (BarthezTokenizer, BarthezTokenizerFast)), (MBartConfig, (MBartTokenizer, MBartTokenizerFast)), (XLMRobertaConfig, (XLMRobertaTokenizer, XLMRobertaTokenizerFast)), (MarianConfig, (MarianTokenizer, None)), @@ -243,6 +242,7 @@ NO_CONFIG_TOKENIZER = [ HerbertTokenizer, HerbertTokenizerFast, PhobertTokenizer, + BarthezTokenizer, ]