From 562b998366c7a4a2bd0addf1a860fbee0aa04d74 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Guillem=20Garc=C3=ADa=20Subies?=
 <37592763+GuillemGSubies@users.noreply.github.com>
Date: Tue, 20 Aug 2019 14:10:19 +0200
Subject: [PATCH] Update tokenization_openai.py

---
 pytorch_transformers/tokenization_openai.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pytorch_transformers/tokenization_openai.py b/pytorch_transformers/tokenization_openai.py
index 0f6a8f1dae..79eb023a8d 100644
--- a/pytorch_transformers/tokenization_openai.py
+++ b/pytorch_transformers/tokenization_openai.py
@@ -92,6 +92,7 @@ class OpenAIGPTTokenizer(PreTrainedTokenizer):
             from spacy.lang.en import English
             _nlp = English()
             self.nlp = nlp.Defaults.create_tokenizer(_nlp)
+            self.fix_text = ftfy.fix_text
         except ImportError:
             logger.warning("ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.")
             self.nlp = BasicTokenizer(do_lower_case=True)