SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Merge pull request #2312 from vitaliyradchenko/fix_special_and_add_tokens_loading

Browse Source 
Correct tokenization for special and added tokens
This commit is contained in:
Thomas Wolf
2019-12-25 20:52:30 +01:00
committed by GitHub
parent cea04a2443 b262577d17
commit 1240be3ed9
1 changed files with 4 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/transformers/tokenization_utils.py
View File

@@ -469,6 +469,9 @@ class PreTrainedTokenizer(object):
tokenizer.init_inputs = init_inputs
tokenizer.init_kwargs = init_kwargs
# update unique_added_tokens_encoder with special tokens for correct tokenization
tokenizer.unique_added_tokens_encoder.update(set(tokenizer.all_special_tokens))
# Add supplementary tokens.
if added_tokens_file is not None:
with open(added_tokens_file, encoding="utf-8") as added_tokens_handle:
@@ -476,6 +479,7 @@ class PreTrainedTokenizer(object):
added_tok_decoder = {v: k for k, v in added_tok_encoder.items()}
tokenizer.added_tokens_encoder.update(added_tok_encoder)
tokenizer.added_tokens_decoder.update(added_tok_decoder)
tokenizer.unique_added_tokens_encoder.update(set(tokenizer.added_tokens_encoder.keys()))
return tokenizer