From a15562e17059ce00a9a376049f2195020ecd392e Mon Sep 17 00:00:00 2001 From: Shijie Wu Date: Tue, 3 Sep 2019 18:27:29 -0700 Subject: [PATCH] Fix reference of import when called for the second time --- pytorch_transformers/tokenization_xlm.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pytorch_transformers/tokenization_xlm.py b/pytorch_transformers/tokenization_xlm.py index 8155413f75..f7231384b3 100644 --- a/pytorch_transformers/tokenization_xlm.py +++ b/pytorch_transformers/tokenization_xlm.py @@ -705,6 +705,8 @@ class XLMTokenizer(PreTrainedTokenizer): try: if 'pythainlp' not in sys.modules: from pythainlp.tokenize import word_tokenize as th_word_tokenize + else: + th_word_tokenize = sys.modules['pythainlp'].word_tokenize except (AttributeError, ImportError) as e: logger.error("Make sure you install PyThaiNLP (https://github.com/PyThaiNLP/pythainlp) with the following steps") logger.error("1. pip install pythainlp") @@ -714,6 +716,8 @@ class XLMTokenizer(PreTrainedTokenizer): try: if 'jieba' not in sys.modules: import jieba + else: + jieba = sys.modules['jieba'] except (AttributeError, ImportError) as e: logger.error("Make sure you install Jieba (https://github.com/fxsjy/jieba) with the following steps") logger.error("1. pip install jieba")