SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

FastPreTrainedTokenizer => PreTrainedTokenizerFast

Browse Source
This commit is contained in:
Anthony MOI
2019-12-26 14:39:39 -05:00
parent 1f82a5d910
commit 7ead04ce14
3 changed files with 6 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/transformers/tokenization_bert.py
View File

@@ -22,7 +22,7 @@ import unicodedata
import tokenizers as tk
from .tokenization_utils import FastPreTrainedTokenizer, PreTrainedTokenizer
from .tokenization_utils import PreTrainedTokenizerFast, PreTrainedTokenizer
logger = logging.getLogger(__name__)
@@ -529,7 +529,7 @@ def _is_punctuation(char):
return False
class BertTokenizerFast(FastPreTrainedTokenizer):
class BertTokenizerFast(PreTrainedTokenizerFast):
vocab_files_names = VOCAB_FILES_NAMES
pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION