Added binary masking tests

2019-09-02 17:47:16 -04:00
parent e391d4735e
commit c3df2136e1
2 changed files with 14 additions and 0 deletions
--- a/pytorch_transformers/tokenization_utils.py
+++ b/pytorch_transformers/tokenization_utils.py
@@ -690,6 +690,8 @@ class PreTrainedTokenizer(object):
        if add_special_tokens:
            return self.add_special_tokens_sentences_pair(first_sentence_tokens, second_sentence_tokens, output_mask)
        else:
+            if output_mask:
+                logger.warning("Can't output mask if no special tokens are involved. Please call the method with add_special_tokens set to True.")
            return first_sentence_tokens, second_sentence_tokens

    def add_special_tokens_single_sentence(self, token_ids):