SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Fix interaction of return_token_type_ids and add_special_tokens (#8854)

Browse Source
This commit is contained in:
Lysandre Debut
2020-12-08 12:04:01 -05:00
committed by GitHub
parent 04c446f764
commit b7cdd00f15
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/transformers/tokenization_utils_base.py
View File

@@ -2814,7 +2814,7 @@ class PreTrainedTokenizerBase(SpecialTokensMixin):
len_ids = len(ids) len_ids = len(ids)
len_pair_ids = len(pair_ids) if pair else 0 len_pair_ids = len(pair_ids) if pair else 0
if return_token_type_ids is not None and not add_special_tokens: if return_token_type_ids and not add_special_tokens:
raise ValueError( raise ValueError(
"Asking to return token_type_ids while setting add_special_tokens to False " "Asking to return token_type_ids while setting add_special_tokens to False "
"results in an undefined behavior. Please set add_special_tokens to True or " "results in an undefined behavior. Please set add_special_tokens to True or "