Fix interaction of return_token_type_ids and add_special_tokens (#8854)

2020-12-08 12:04:01 -05:00
parent 04c446f764
commit b7cdd00f15
1 changed files with 1 additions and 1 deletions
--- a/src/transformers/tokenization_utils_base.py
+++ b/src/transformers/tokenization_utils_base.py
@@ -2814,7 +2814,7 @@ class PreTrainedTokenizerBase(SpecialTokensMixin):
        len_ids = len(ids)
        len_pair_ids = len(pair_ids) if pair else 0

-        if return_token_type_ids is not None and not add_special_tokens:
+        if return_token_type_ids and not add_special_tokens:
            raise ValueError(
                "Asking to return token_type_ids while setting add_special_tokens to False "
                "results in an undefined behavior. Please set add_special_tokens to True or "