good quality generation example for GPT, GPT-2, Transfo-XL, XLNet

2019-07-13 15:25:03 +02:00
parent 7322c314a6
commit 7d4b200e40
10 changed files with 252 additions and 46 deletions
--- a/pytorch_transformers/tokenization_xlnet.py
+++ b/pytorch_transformers/tokenization_xlnet.py
@@ -172,7 +172,7 @@ class XLNetTokenizer(PreTrainedTokenizer):

    def _convert_ids_to_string(self, tokens_ids):
        """Converts a sequence of ids in a string."""
-        out_string = ''.join(tokens_ids)
+        out_string = ''.join(tokens_ids).replace(SPIECE_UNDERLINE, ' ')
        return out_string

    def save_vocabulary(self, save_directory):