Appends space before special token

2019-09-01 21:07:00 -04:00
parent fede4ef45d
commit 31d3373bc9
1 changed files with 1 additions and 1 deletions
--- a/pytorch_transformers/tokenization_utils.py
+++ b/pytorch_transformers/tokenization_utils.py
@@ -760,7 +760,7 @@ class PreTrainedTokenizer(object):
                if current_sub_text:
                    sub_texts.append(self.convert_tokens_to_string(current_sub_text))
                    current_sub_text = []
-                sub_texts.append(token)
+                sub_texts.append(" " + token)
            else:
                current_sub_text.append(token)
        if current_sub_text: