SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

fixing tokenization of extra_id symbols in T5Tokenizer. Related to issue 4021 (#4353)

Browse Source
This commit is contained in:
Elman Mansimov
2020-05-25 16:04:30 -04:00
committed by GitHub
parent 5139733623
commit 3dea40b858
1 changed files with 1 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/transformers/tokenization_utils.py
View File

@@ -503,6 +503,7 @@ class SpecialTokensMixin:
if key in self.SPECIAL_TOKENS_ATTRIBUTES: if key in self.SPECIAL_TOKENS_ATTRIBUTES:
if key == "additional_special_tokens": if key == "additional_special_tokens":
assert isinstance(value, (list, tuple)) and all(isinstance(t, str) for t in value) assert isinstance(value, (list, tuple)) and all(isinstance(t, str) for t in value)
setattr(self, key, value)
elif isinstance(value, AddedTokenFast): elif isinstance(value, AddedTokenFast):
setattr(self, key, str(value)) setattr(self, key, str(value))
elif isinstance(value, str): elif isinstance(value, str):