SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Add missing arguments for BertWordPieceTokenizer (#5810)

Browse Source
This commit is contained in:
Jangwon Park
2020-09-07 21:35:41 +09:00
committed by GitHub
parent 77cd0e13d2
commit 90ec78b514
1 changed files with 2 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/transformers/tokenization_bert.py
View File

@@ -632,6 +632,8 @@ class BertTokenizerFast(PreTrainedTokenizerFast):
unk_token=unk_token, unk_token=unk_token,
sep_token=sep_token, sep_token=sep_token,
cls_token=cls_token, cls_token=cls_token,
pad_token=pad_token,
mask_token=mask_token,
clean_text=clean_text, clean_text=clean_text,
handle_chinese_chars=tokenize_chinese_chars, handle_chinese_chars=tokenize_chinese_chars,
strip_accents=strip_accents, strip_accents=strip_accents,