Fix special tokens addition in decoder

2019-11-12 15:27:57 -05:00
parent d409aca326
commit 74d0bcb6ff
2 changed files with 21 additions and 1 deletions
--- a/transformers/tokenization_utils.py
+++ b/transformers/tokenization_utils.py
@@ -1055,7 +1055,7 @@ class PreTrainedTokenizer(object):
            class attributes (cls_token, unk_token...).
        """
        all_toks = self.all_special_tokens
-        all_ids = list(self._convert_token_to_id(t) for t in all_toks)
+        all_ids = self.convert_tokens_to_ids(all_toks)
        return all_ids

    @staticmethod