fix adding special tokens

This commit is contained in:
epwalsh
2019-08-29 13:44:50 -07:00
parent caf1d116a6
commit 07e21307b6

View File

@@ -676,7 +676,7 @@ class PreTrainedTokenizer(object):
all_toks = []
set_attr = self.special_tokens_map
for attr_value in set_attr.values():
all_toks = all_toks + (attr_value if isinstance(attr_value, (list, tuple)) else [attr_value])
all_toks = all_toks + (list(attr_value) if isinstance(attr_value, (list, tuple)) else [attr_value])
all_toks = list(set(all_toks))
return all_toks