From 31d3373bc92d93165bfb0788936939e23a7c5b73 Mon Sep 17 00:00:00 2001
From: LysandreJik <lysandre.debut@reseau.eseo.fr>
Date: Sun, 1 Sep 2019 21:07:00 -0400
Subject: [PATCH] Appends space before special token

---
 pytorch_transformers/tokenization_utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pytorch_transformers/tokenization_utils.py b/pytorch_transformers/tokenization_utils.py
index 4b52409eea..53b8d245b8 100644
--- a/pytorch_transformers/tokenization_utils.py
+++ b/pytorch_transformers/tokenization_utils.py
@@ -760,7 +760,7 @@ class PreTrainedTokenizer(object):
                 if current_sub_text:
                     sub_texts.append(self.convert_tokens_to_string(current_sub_text))
                     current_sub_text = []
-                sub_texts.append(token)
+                sub_texts.append(" " + token)
             else:
                 current_sub_text.append(token)
         if current_sub_text: