Update tokenization_openai.py

2019-08-20 14:10:19 +02:00
parent bb04446285
commit 562b998366
1 changed files with 1 additions and 0 deletions
--- a/pytorch_transformers/tokenization_openai.py
+++ b/pytorch_transformers/tokenization_openai.py
@@ -92,6 +92,7 @@ class OpenAIGPTTokenizer(PreTrainedTokenizer):
            from spacy.lang.en import English
            _nlp = English()
            self.nlp = nlp.Defaults.create_tokenizer(_nlp)
+            self.fix_text = ftfy.fix_text
        except ImportError:
            logger.warning("ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.")
            self.nlp = BasicTokenizer(do_lower_case=True)