From bb04446285be43059050406b3bc4938807c63c25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guillem=20Garc=C3=ADa=20Subies?= <37592763+GuillemGSubies@users.noreply.github.com> Date: Tue, 20 Aug 2019 14:07:40 +0200 Subject: [PATCH] Update tokenization_openai.py --- pytorch_transformers/tokenization_openai.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pytorch_transformers/tokenization_openai.py b/pytorch_transformers/tokenization_openai.py index 0eb5281d39..0f6a8f1dae 100644 --- a/pytorch_transformers/tokenization_openai.py +++ b/pytorch_transformers/tokenization_openai.py @@ -89,9 +89,9 @@ class OpenAIGPTTokenizer(PreTrainedTokenizer): try: import ftfy - import spacy - self.nlp = spacy.load('en', disable=['parser', 'tagger', 'ner', 'textcat']) - self.fix_text = ftfy.fix_text + from spacy.lang.en import English + _nlp = English() + self.nlp = nlp.Defaults.create_tokenizer(_nlp) except ImportError: logger.warning("ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.") self.nlp = BasicTokenizer(do_lower_case=True)