From 90ec78b5140251f093f658ebd4d2925e8c03f5e6 Mon Sep 17 00:00:00 2001 From: Jangwon Park Date: Mon, 7 Sep 2020 21:35:41 +0900 Subject: [PATCH] Add missing arguments for BertWordPieceTokenizer (#5810) --- src/transformers/tokenization_bert.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/transformers/tokenization_bert.py b/src/transformers/tokenization_bert.py index ae7ae6ea99..98f8287e34 100644 --- a/src/transformers/tokenization_bert.py +++ b/src/transformers/tokenization_bert.py @@ -632,6 +632,8 @@ class BertTokenizerFast(PreTrainedTokenizerFast): unk_token=unk_token, sep_token=sep_token, cls_token=cls_token, + pad_token=pad_token, + mask_token=mask_token, clean_text=clean_text, handle_chinese_chars=tokenize_chinese_chars, strip_accents=strip_accents,