SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Moved duplicate token check

Browse Source
This commit is contained in:
danai-antoniou
2019-10-02 07:44:15 +01:00
parent d73957899a
commit a95158518d
1 changed files with 2 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
transformers/tokenization_utils.py
View File

@@ -508,14 +508,12 @@ class PreTrainedTokenizer(object):
if not new_tokens: if not new_tokens:
return 0 return 0
if len(new_tokens) != len(set(new_tokens)):
raise ValueError("The provided list of tokens contains duplicates.")
to_add_tokens = [] to_add_tokens = []
for token in new_tokens: for token in new_tokens:
assert isinstance(token, str) or (six.PY2 and isinstance(token, unicode)) assert isinstance(token, str) or (six.PY2 and isinstance(token, unicode))
if token != self.unk_token and \ if token != self.unk_token and \
self.convert_tokens_to_ids(token) == self.convert_tokens_to_ids(self.unk_token): self.convert_tokens_to_ids(token) == self.convert_tokens_to_ids(self.unk_token) and \
token not in to_add_tokens:
to_add_tokens.append(token) to_add_tokens.append(token)
logger.info("Adding %s to the vocabulary", token) logger.info("Adding %s to the vocabulary", token)