From 850da1cc36f95175219420365ac3fb95b483ce8d Mon Sep 17 00:00:00 2001 From: thomwolf Date: Mon, 4 Feb 2019 17:35:05 +0100 Subject: [PATCH] strip decoded outputs --- pytorch_pretrained_bert/tokenization_openai.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_pretrained_bert/tokenization_openai.py b/pytorch_pretrained_bert/tokenization_openai.py index e545e0d375..aba531caed 100644 --- a/pytorch_pretrained_bert/tokenization_openai.py +++ b/pytorch_pretrained_bert/tokenization_openai.py @@ -228,5 +228,5 @@ class OpenAIGPTTokenizer(object): def decode(self, ids, skip_special_tokens=False): """Converts a sequence of ids in a string.""" tokens = self.convert_ids_to_tokens(ids, skip_special_tokens=skip_special_tokens) - out_string = ''.join(tokens).replace('', ' ') + out_string = ''.join(tokens).replace('', ' ').strip() return out_string