Added documentation and changed parameters for special_tokens_sentences_pair.

2019-08-12 15:13:53 -04:00
parent 912fdff899
commit 22ac004a7c
5 changed files with 50 additions and 18 deletions
--- a/pytorch_transformers/tokenization_utils.py
+++ b/pytorch_transformers/tokenization_utils.py
@@ -546,7 +546,7 @@ class PreTrainedTokenizer(object):
    def add_special_tokens_single_sentence(self, token_ids):
        raise NotImplementedError

-    def add_special_tokens_sentences_pair(self, *token_ids):
+    def add_special_tokens_sentences_pair(self, token_ids_0, token_ids_1):
        raise NotImplementedError

    def convert_ids_to_tokens(self, ids, skip_special_tokens=False):