SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Remove annoying tokenization message

Browse Source
This commit is contained in:
LysandreJik
2019-12-02 18:23:00 -05:00
parent e85855f2c4
commit fbaf05bd92
1 changed files with 0 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
transformers/tokenization_utils.py
View File

@@ -981,7 +981,6 @@ class PreTrainedTokenizer(object):
return (ids, pair_ids, overflowing_tokens) return (ids, pair_ids, overflowing_tokens)
def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1=None): def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1=None):
logger.warning("This tokenizer does not make use of special tokens.")
if token_ids_1 is None: if token_ids_1 is None:
return len(token_ids_0) * [0] return len(token_ids_0) * [0]
return [0] * len(token_ids_0) + [1] * len(token_ids_1) return [0] * len(token_ids_0) + [1] * len(token_ids_1)
@@ -994,7 +993,6 @@ class PreTrainedTokenizer(object):
single sequence: <s> X </s> single sequence: <s> X </s>
pair of sequences: <s> A </s></s> B </s> pair of sequences: <s> A </s></s> B </s>
""" """
logger.warning("This tokenizer does not make use of special tokens. Input is returned with no modification.")
if token_ids_1 is None: if token_ids_1 is None:
return token_ids_0 return token_ids_0
return token_ids_0 + token_ids_1 return token_ids_0 + token_ids_1