SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

fix #2031

Browse Source
This commit is contained in:
thomwolf
2019-12-05 14:06:29 +01:00
parent af077b15e2
commit 2d5d86e037
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
transformers/tokenization_albert.py
View File

@@ -66,7 +66,7 @@ class AlbertTokenizer(PreTrainedTokenizer):
def __init__(self, vocab_file, def __init__(self, vocab_file,
do_lower_case=True, remove_space=True, keep_accents=False, do_lower_case=True, remove_space=True, keep_accents=False,
bos_token="[CLS]", eos_token="[SEP]", unk_token="<unk>", sep_token="[SEP]", bos_token="[CLS]", eos_token="[SEP]", unk_token="<unk>", sep_token="[SEP]",
pad_token="<pad>", cls_token="[CLS]", mask_token="[MASK]>", **kwargs): pad_token="<pad>", cls_token="[CLS]", mask_token="[MASK]", **kwargs):
super(AlbertTokenizer, self).__init__(bos_token=bos_token, eos_token=eos_token, super(AlbertTokenizer, self).__init__(bos_token=bos_token, eos_token=eos_token,
unk_token=unk_token, sep_token=sep_token, unk_token=unk_token, sep_token=sep_token,
pad_token=pad_token, cls_token=cls_token, pad_token=pad_token, cls_token=cls_token,