From 07e21307b6b16350425ea08790c7e20c081a3b3f Mon Sep 17 00:00:00 2001 From: epwalsh Date: Thu, 29 Aug 2019 13:44:50 -0700 Subject: [PATCH] fix adding special tokens --- pytorch_transformers/tokenization_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_transformers/tokenization_utils.py b/pytorch_transformers/tokenization_utils.py index 9d814f704f..588ff8ab66 100644 --- a/pytorch_transformers/tokenization_utils.py +++ b/pytorch_transformers/tokenization_utils.py @@ -676,7 +676,7 @@ class PreTrainedTokenizer(object): all_toks = [] set_attr = self.special_tokens_map for attr_value in set_attr.values(): - all_toks = all_toks + (attr_value if isinstance(attr_value, (list, tuple)) else [attr_value]) + all_toks = all_toks + (list(attr_value) if isinstance(attr_value, (list, tuple)) else [attr_value]) all_toks = list(set(all_toks)) return all_toks