Merge pull request #1811 from huggingface/special-tokens

Fix special tokens addition in decoder #1807
2019-11-14 22:17:24 +01:00
parent 1a237d7f42 74d0bcb6ff
commit 5b322a36db
2 changed files with 21 additions and 1 deletions
--- a/transformers/tokenization_utils.py
+++ b/transformers/tokenization_utils.py
@@ -1057,7 +1057,7 @@ class PreTrainedTokenizer(object):
            class attributes (cls_token, unk_token...).
        """
        all_toks = self.all_special_tokens
-        all_ids = list(self._convert_token_to_id(t) for t in all_toks)
+        all_ids = self.convert_tokens_to_ids(all_toks)
        return all_ids

    @staticmethod