From 5d912e7ed4d3b61e69e1b93ab465982b98eedb30 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Sat, 4 Apr 2020 15:04:03 -0400 Subject: [PATCH] Tweak typing for #3566 --- src/transformers/tokenization_bert_japanese.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/transformers/tokenization_bert_japanese.py b/src/transformers/tokenization_bert_japanese.py index d370e0dced..6c91240435 100644 --- a/src/transformers/tokenization_bert_japanese.py +++ b/src/transformers/tokenization_bert_japanese.py @@ -19,6 +19,7 @@ import collections import logging import os import unicodedata +from typing import Optional from .tokenization_bert import BasicTokenizer, BertTokenizer, WordpieceTokenizer, load_vocab @@ -165,10 +166,10 @@ class BertJapaneseTokenizer(BertTokenizer): return split_tokens -class MecabTokenizer(object): +class MecabTokenizer: """Runs basic tokenization with MeCab morphological parser.""" - def __init__(self, do_lower_case=False, never_split=None, normalize_text=True, mecab_option=None): + def __init__(self, do_lower_case=False, never_split=None, normalize_text=True, mecab_option: Optional[str] = None): """Constructs a MecabTokenizer. Args: