From 2ba1a14fb0586b9ce61769a8341ecfbfbc8a1507 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Thu, 22 Aug 2019 17:25:55 -0400 Subject: [PATCH] Decode now calls private property instead of public method --- pytorch_transformers/tokenization_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pytorch_transformers/tokenization_utils.py b/pytorch_transformers/tokenization_utils.py index d2855e0922..d4cbd85d67 100644 --- a/pytorch_transformers/tokenization_utils.py +++ b/pytorch_transformers/tokenization_utils.py @@ -636,9 +636,9 @@ class PreTrainedTokenizer(object): filtered_tokens = self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens) text = self.convert_tokens_to_string(filtered_tokens) - if self.sep_token is not None and self.sep_token in text: - text = text.replace(self.cls_token, self.sep_token) - split_text = list(filter(lambda sentence: len(sentence) > 0, text.split(self.sep_token))) + if self._sep_token is not None and self._sep_token in text: + text = text.replace(self._cls_token, self._sep_token) + split_text = list(filter(lambda sentence: len(sentence) > 0, text.split(self._sep_token))) if clean_up_tokenization_spaces: clean_text = [self.clean_up_tokenization(text) for text in split_text] return clean_text