Supports already existing special tokens

transformers/tokenization_xlnet.py

@@ -200,7 +200,7 @@ class XLNetTokenizer(PreTrainedTokenizer):
         cls = [self.cls_token_id]
         return token_ids_0 + sep + token_ids_1 + sep + cls
 
-    def get_sequence_ids(self, token_ids_0, token_ids_1=None):
+    def get_sequence_ids(self, token_ids_0, token_ids_1=None, special_tokens_present=False):
         """
         Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding
         special tokens using the tokenizer ``prepare_for_model`` or ``encode_plus`` methods.
@@ -213,6 +213,10 @@ class XLNetTokenizer(PreTrainedTokenizer):
         Returns:
             A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token.
         """
+
+        if special_tokens_present:
+            return list(map(lambda x: 0 if x in [self.sep_token_id, self.cls_token_id] else 1, token_ids_0))
+
         if token_ids_1:
             return ([1] * len(token_ids_0)) + [0] + ([1] * len(token_ids_1)) + [0, 0]
         else: