SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Fix reference of import when called for the second time

Browse Source
This commit is contained in:
Shijie Wu
2019-09-03 18:27:29 -07:00
committed by GitHub
parent 0287d264e9
commit a15562e170
1 changed files with 4 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
pytorch_transformers/tokenization_xlm.py
View File

@@ -705,6 +705,8 @@ class XLMTokenizer(PreTrainedTokenizer):
try: try:
if 'pythainlp' not in sys.modules: if 'pythainlp' not in sys.modules:
from pythainlp.tokenize import word_tokenize as th_word_tokenize from pythainlp.tokenize import word_tokenize as th_word_tokenize
else:
th_word_tokenize = sys.modules['pythainlp'].word_tokenize
except (AttributeError, ImportError) as e: except (AttributeError, ImportError) as e:
logger.error("Make sure you install PyThaiNLP (https://github.com/PyThaiNLP/pythainlp) with the following steps") logger.error("Make sure you install PyThaiNLP (https://github.com/PyThaiNLP/pythainlp) with the following steps")
logger.error("1. pip install pythainlp") logger.error("1. pip install pythainlp")
@@ -714,6 +716,8 @@ class XLMTokenizer(PreTrainedTokenizer):
try: try:
if 'jieba' not in sys.modules: if 'jieba' not in sys.modules:
import jieba import jieba
else:
jieba = sys.modules['jieba']
except (AttributeError, ImportError) as e: except (AttributeError, ImportError) as e:
logger.error("Make sure you install Jieba (https://github.com/fxsjy/jieba) with the following steps") logger.error("Make sure you install Jieba (https://github.com/fxsjy/jieba) with the following steps")
logger.error("1. pip install jieba") logger.error("1. pip install jieba")