SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Reraise ImportError when sentencepiece isn't installed.

Browse Source 
Else, the next line fails with a confusion exception because the spm
variable isn't defined.
This commit is contained in:
Aymeric Augustin
2019-12-23 21:27:42 +01:00
parent c8b0c1e551
commit a31d4a2971
3 changed files with 6 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/transformers/tokenization_albert.py
View File

@@ -100,6 +100,7 @@ class AlbertTokenizer(PreTrainedTokenizer):
"You need to install SentencePiece to use AlbertTokenizer: https://github.com/google/sentencepiece" "You need to install SentencePiece to use AlbertTokenizer: https://github.com/google/sentencepiece"
"pip install sentencepiece" "pip install sentencepiece"
) )
raise
self.do_lower_case = do_lower_case self.do_lower_case = do_lower_case
self.remove_space = remove_space self.remove_space = remove_space
@@ -127,6 +128,7 @@ class AlbertTokenizer(PreTrainedTokenizer):
"You need to install SentencePiece to use AlbertTokenizer: https://github.com/google/sentencepiece" "You need to install SentencePiece to use AlbertTokenizer: https://github.com/google/sentencepiece"
"pip install sentencepiece" "pip install sentencepiece"
) )
raise
self.sp_model = spm.SentencePieceProcessor() self.sp_model = spm.SentencePieceProcessor()
self.sp_model.Load(self.vocab_file) self.sp_model.Load(self.vocab_file)

2
src/transformers/tokenization_t5.py
View File

@@ -107,6 +107,7 @@ class T5Tokenizer(PreTrainedTokenizer):
"https://github.com/google/sentencepiece" "https://github.com/google/sentencepiece"
"pip install sentencepiece" "pip install sentencepiece"
) )
raise
self.vocab_file = vocab_file self.vocab_file = vocab_file
self._extra_ids = extra_ids self._extra_ids = extra_ids
@@ -132,6 +133,7 @@ class T5Tokenizer(PreTrainedTokenizer):
"You need to install SentencePiece to use XLNetTokenizer: https://github.com/google/sentencepiece" "You need to install SentencePiece to use XLNetTokenizer: https://github.com/google/sentencepiece"
"pip install sentencepiece" "pip install sentencepiece"
) )
raise
self.sp_model = spm.SentencePieceProcessor() self.sp_model = spm.SentencePieceProcessor()
self.sp_model.Load(self.vocab_file) self.sp_model.Load(self.vocab_file)

2
src/transformers/tokenization_xlnet.py
View File

@@ -100,6 +100,7 @@ class XLNetTokenizer(PreTrainedTokenizer):
"You need to install SentencePiece to use XLNetTokenizer: https://github.com/google/sentencepiece" "You need to install SentencePiece to use XLNetTokenizer: https://github.com/google/sentencepiece"
"pip install sentencepiece" "pip install sentencepiece"
) )
raise
self.do_lower_case = do_lower_case self.do_lower_case = do_lower_case
self.remove_space = remove_space self.remove_space = remove_space
@@ -127,6 +128,7 @@ class XLNetTokenizer(PreTrainedTokenizer):
"You need to install SentencePiece to use XLNetTokenizer: https://github.com/google/sentencepiece" "You need to install SentencePiece to use XLNetTokenizer: https://github.com/google/sentencepiece"
"pip install sentencepiece" "pip install sentencepiece"
) )
raise
self.sp_model = spm.SentencePieceProcessor() self.sp_model = spm.SentencePieceProcessor()
self.sp_model.Load(self.vocab_file) self.sp_model.Load(self.vocab_file)