From a31d4a2971abe1f349a6181dce1010368a822355 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Mon, 23 Dec 2019 21:27:42 +0100 Subject: [PATCH] Reraise ImportError when sentencepiece isn't installed. Else, the next line fails with a confusion exception because the spm variable isn't defined. --- src/transformers/tokenization_albert.py | 2 ++ src/transformers/tokenization_t5.py | 2 ++ src/transformers/tokenization_xlnet.py | 2 ++ 3 files changed, 6 insertions(+) diff --git a/src/transformers/tokenization_albert.py b/src/transformers/tokenization_albert.py index 541ae7ae68..0c06b48a49 100644 --- a/src/transformers/tokenization_albert.py +++ b/src/transformers/tokenization_albert.py @@ -100,6 +100,7 @@ class AlbertTokenizer(PreTrainedTokenizer): "You need to install SentencePiece to use AlbertTokenizer: https://github.com/google/sentencepiece" "pip install sentencepiece" ) + raise self.do_lower_case = do_lower_case self.remove_space = remove_space @@ -127,6 +128,7 @@ class AlbertTokenizer(PreTrainedTokenizer): "You need to install SentencePiece to use AlbertTokenizer: https://github.com/google/sentencepiece" "pip install sentencepiece" ) + raise self.sp_model = spm.SentencePieceProcessor() self.sp_model.Load(self.vocab_file) diff --git a/src/transformers/tokenization_t5.py b/src/transformers/tokenization_t5.py index bd7a6e3144..02c70fe5da 100644 --- a/src/transformers/tokenization_t5.py +++ b/src/transformers/tokenization_t5.py @@ -107,6 +107,7 @@ class T5Tokenizer(PreTrainedTokenizer): "https://github.com/google/sentencepiece" "pip install sentencepiece" ) + raise self.vocab_file = vocab_file self._extra_ids = extra_ids @@ -132,6 +133,7 @@ class T5Tokenizer(PreTrainedTokenizer): "You need to install SentencePiece to use XLNetTokenizer: https://github.com/google/sentencepiece" "pip install sentencepiece" ) + raise self.sp_model = spm.SentencePieceProcessor() self.sp_model.Load(self.vocab_file) diff --git a/src/transformers/tokenization_xlnet.py b/src/transformers/tokenization_xlnet.py index 6d0a6d0f18..38c2a00ff9 100644 --- a/src/transformers/tokenization_xlnet.py +++ b/src/transformers/tokenization_xlnet.py @@ -100,6 +100,7 @@ class XLNetTokenizer(PreTrainedTokenizer): "You need to install SentencePiece to use XLNetTokenizer: https://github.com/google/sentencepiece" "pip install sentencepiece" ) + raise self.do_lower_case = do_lower_case self.remove_space = remove_space @@ -127,6 +128,7 @@ class XLNetTokenizer(PreTrainedTokenizer): "You need to install SentencePiece to use XLNetTokenizer: https://github.com/google/sentencepiece" "pip install sentencepiece" ) + raise self.sp_model = spm.SentencePieceProcessor() self.sp_model.Load(self.vocab_file)