This commit is contained in:
maru0kun
2019-09-05 19:24:57 +09:00
committed by GitHub
parent 0b52642d37
commit d737947725

View File

@@ -174,7 +174,7 @@ class PreTrainedTokenizer(object):
return self.convert_tokens_to_ids(self._eos_token)
@property
def unk_token_is(self):
def unk_token_id(self):
""" Id of the unknown token in the vocabulary. Log an error if used while not having been set. """
if self._unk_token is None:
logger.error("Using unk_token, but it is not set yet.")