SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Merge pull request #1152 from epwalsh/fix-special-tokens

Browse Source 
fix adding special tokens
This commit is contained in:
Thomas Wolf
2019-08-30 23:21:59 +02:00
committed by GitHub
parent d483cd8e46 07e21307b6
commit f3d18c71ec
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
pytorch_transformers/tokenization_utils.py
View File

@@ -784,7 +784,7 @@ class PreTrainedTokenizer(object):
all_toks = [] all_toks = []
set_attr = self.special_tokens_map set_attr = self.special_tokens_map
for attr_value in set_attr.values(): for attr_value in set_attr.values():
all_toks = all_toks + (attr_value if isinstance(attr_value, (list, tuple)) else [attr_value]) all_toks = all_toks + (list(attr_value) if isinstance(attr_value, (list, tuple)) else [attr_value])
all_toks = list(set(all_toks)) all_toks = list(set(all_toks))
return all_toks return all_toks