SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Tweak typing for #3566

Browse Source
This commit is contained in:
Julien Chaumond
2020-04-04 15:04:03 -04:00
parent 94eb68d742
commit 5d912e7ed4
1 changed files with 3 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/transformers/tokenization_bert_japanese.py
View File

@@ -19,6 +19,7 @@ import collections
import logging import logging
import os import os
import unicodedata import unicodedata
from typing import Optional
from .tokenization_bert import BasicTokenizer, BertTokenizer, WordpieceTokenizer, load_vocab from .tokenization_bert import BasicTokenizer, BertTokenizer, WordpieceTokenizer, load_vocab
@@ -165,10 +166,10 @@ class BertJapaneseTokenizer(BertTokenizer):
return split_tokens return split_tokens
class MecabTokenizer(object): class MecabTokenizer:
"""Runs basic tokenization with MeCab morphological parser.""" """Runs basic tokenization with MeCab morphological parser."""
def __init__(self, do_lower_case=False, never_split=None, normalize_text=True, mecab_option=None): def __init__(self, do_lower_case=False, never_split=None, normalize_text=True, mecab_option: Optional[str] = None):
"""Constructs a MecabTokenizer. """Constructs a MecabTokenizer.
Args: Args: