From bfe93a5a21a77e0a0f9b35132810aab9d0b1f04c Mon Sep 17 00:00:00 2001 From: thomwolf Date: Fri, 30 Aug 2019 22:43:26 +0200 Subject: [PATCH] fix distilbert in auto tokenizer --- pytorch_transformers/tokenization_auto.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pytorch_transformers/tokenization_auto.py b/pytorch_transformers/tokenization_auto.py index 357d184e58..889774b36c 100644 --- a/pytorch_transformers/tokenization_auto.py +++ b/pytorch_transformers/tokenization_auto.py @@ -94,13 +94,13 @@ class AutoTokenizer(object): Examples:: - config = AutoTokenizer.from_pretrained('bert-base-uncased') # Download vocabulary from S3 and cache. - config = AutoTokenizer.from_pretrained('./test/bert_saved_model/') # E.g. tokenizer was saved using `save_pretrained('./test/saved_model/')` + tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased') # Download vocabulary from S3 and cache. + tokenizer = AutoTokenizer.from_pretrained('./test/bert_saved_model/') # E.g. tokenizer was saved using `save_pretrained('./test/saved_model/')` """ if 'distilbert' in pretrained_model_name_or_path: return DistilBertTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) - if 'roberta' in pretrained_model_name_or_path: + elif 'roberta' in pretrained_model_name_or_path: return RobertaTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) elif 'bert' in pretrained_model_name_or_path: return BertTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)